{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_p9IwEC-zmUQ"
   },
   "source": [
    "# 双节倒立摆 Acrobot-v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y7teSwpgzmUS"
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import scipy\n",
    "import matplotlib.pyplot as plt\n",
    "import gym\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 105
    },
    "colab_type": "code",
    "id": "oD2OLrzXai3k",
    "outputId": "89017516-5374-45c1-be0e-4db5afaa8122"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = gym.make('Acrobot-v1')\n",
    "env.seed(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kZMR1c3dkLwp"
   },
   "source": [
    "### 用简单的执行者评论家算法寻找最优策略"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QActorCriticAgent:\n",
    "    def __init__(self, env, actor_kwargs, critic_kwargs, gamma=0.99):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = gamma\n",
    "        self.discount = 1.\n",
    "        \n",
    "        self.actor_net = self.build_network(output_size=self.action_n,\n",
    "                output_activation=tf.nn.softmax,\n",
    "                loss=tf.losses.categorical_crossentropy,\n",
    "                **actor_kwargs)\n",
    "        self.critic_net = self.build_network(output_size=self.action_n,\n",
    "                **critic_kwargs)\n",
    "    \n",
    "    def build_network(self, hidden_sizes, output_size, input_size=None,\n",
    "                activation=tf.nn.relu, output_activation=None,\n",
    "                loss=tf.losses.mse, learning_rate=0.01):\n",
    "        model = keras.Sequential()\n",
    "        for idx, hidden_size in enumerate(hidden_sizes):\n",
    "            kwargs = {}\n",
    "            if idx == 0 and input_size is not None:\n",
    "                kwargs['input_shape'] = (input_size,)\n",
    "            model.add(keras.layers.Dense(units=hidden_size,\n",
    "                    activation=activation, **kwargs))\n",
    "        model.add(keras.layers.Dense(units=output_size,\n",
    "                activation=output_activation))\n",
    "        optimizer = tf.optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "    \n",
    "    def decide(self, observation):\n",
    "        probs = self.actor_net.predict(observation[np.newaxis])[0]\n",
    "        action = np.random.choice(self.action_n, p=probs)\n",
    "        return action\n",
    "    \n",
    "    def learn(self, observation, action, reward, next_observation,\n",
    "            done, next_action=None):\n",
    "        # 训练执行者网络\n",
    "        x = observation[np.newaxis]\n",
    "        u = self.critic_net.predict(x)\n",
    "        q = u[0, action]\n",
    "        x_tensor = tf.convert_to_tensor(x, dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            pi_tensor = self.actor_net(x_tensor)[0, action]\n",
    "            logpi_tensor = tf.math.log(tf.clip_by_value(pi_tensor,\n",
    "                    1e-6, 1.))\n",
    "            loss_tensor = -self.discount * q * logpi_tensor\n",
    "        grad_tensors = tape.gradient(loss_tensor, self.actor_net.variables)\n",
    "        self.actor_net.optimizer.apply_gradients(zip(\n",
    "                grad_tensors, self.actor_net.variables))\n",
    "        \n",
    "        # 训练评论者网络\n",
    "        u[0, action] = reward\n",
    "        if not done:\n",
    "            q = self.critic_net.predict(\n",
    "                    next_observation[np.newaxis])[0, next_action]\n",
    "            u[0, action] += self.gamma * q\n",
    "        self.critic_net.fit(x, u, verbose=0)\n",
    "        \n",
    "        if done:\n",
    "            self.discount = 1.\n",
    "        else:\n",
    "            self.discount *= self.gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_sarsa(env, agent, train=False, render=False):\n",
    "    episode_reward = 0\n",
    "    observation = env.reset()\n",
    "    action = agent.decide(observation)\n",
    "    while True:\n",
    "        if render:\n",
    "            env.render()\n",
    "        next_observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        if done:\n",
    "            if train:\n",
    "                agent.learn(observation, action, reward,\n",
    "                    next_observation, done)\n",
    "            break\n",
    "        next_action = agent.decide(next_observation)\n",
    "        if train:\n",
    "            agent.learn(observation, action, reward, next_observation,\n",
    "                    done, next_action)\n",
    "        observation, action = next_observation, next_action\n",
    "    return episode_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均回合奖励 = -10256.0 / 100 = -102.56\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXmYHPV55z9vnzOjmdFI6EDXSAKEhMAHsgDbOITLgL1esLPJGjtebJNdnCx+nGTjTeywu3ZiO87GsR1nTZwQhyTEByGJY/TYOBjJGBxzCTAYgW6E7nukufuort/+UVXdPTPdM93T1V1d3e/neeZR96+ruquopr793mKMQVEURWlfIkEfgKIoihIsKgSKoihtjgqBoihKm6NCoCiK0uaoECiKorQ5KgSKoihtjgqBoihKm6NCoCiK0uaoECiKorQ5saAPoBIWLFhgVq1aFfRhKIqihIrnnnvulDFm4UzbhUIIVq1axbPPPhv0YSiKooQKEdlfyXbqGlIURWlzVAgURVHaHBUCRVGUNkeFQFEUpc1RIVAURWlzVAgURVHaHBUCRVGUNkeFQFEUpUFUOxr4X547xANbD9bpaAqoECiKojSAVDbHW//4R3ztx3sr3ucvH9vLgy8eruNROagQKIqiNIAn9p7i6GCKL/5wJ68cGZpx+/2nR9l9YoTr1i2u+7GFosWEoihKM7Dz2DBf/OFOlvZ1ct7COaxf0svGVfMr2nfz9hN0JaJ0JWJ8/J9e5MGPXkk8Wv63+ObtJwC4/iIVAkVRlKbhs99/hWf2DRCLCKOZHAAP3nklb1jRN+1+xhi2bD/OVWsW8p4Ny/jIPzzH1368l49dt6bsPlu2H2fNom76z+ny9RxKoa4hRVGUCnhm3wA/2X2Kj9+wlm1/cCM//vjVRMS5Yc/EtsNDHB9Kc/36xdx48bnc/Ial/L8f7Wb70dIuosHxLM/sG+D69fW3BkCFQFEUZUaMMXzxhztZ2JPkA29eiYiwasEcNvTP49GdJ2fcf/P244jANWudjtCfvvliupMxvvTIrpLbP7brJJZtuP6iRb6eRzlUCBRFUWbgib2neXrfAHdefT6diWh+/Zp1i3jp8CAnhlPT7r95+3E29M/jnO4kAPPnJHjf5f1s2X6cw2fHp2y/Zftx5s9J8MYV8/w9kTKoECiKokyDZw0smdvBrZf3T3jtavcX/mPTWAVHB8d5+cgQ1036df/+K5z3+vbTByasZ3M2j+44wTVrFxGNiB+nMCMqBIqiKMDju07y5Ud2kbMnFn09/PJxnj9wljuvuYCOeHTCa+uX9LKoJ8mPJwnB3pMjZHM2AFvc7J+3T8r+WT6vi2vXLeb+rQfIWHZ+/dnXzjCUshrmFoIahUBEfkVEXhYRW0Q2TnrtkyKyR0R2isiNRes3uWt7ROQTtXy+oiiKX/zRQ9v5ypbd3PWvL+UrgLe+NsBv/+MLXLSkl/+8ccWUfUSEa9Yu4vHdJ7HcG/+/bTvGdV98jKv+5FH+6rG9PPTSUfrnd3HBou4p+/+Xt6zk1EiGH2w7ml/bsv04iWiEX7hwxgmTvlGrRbAN+CXg8eJFEVkP3ApcDNwE/IWIREUkCtwNvANYD7zP3VZRFCUwdh4bZsexYS5e2sv9Ww/yme9tZ9vhQW7/260smdvBfbdfTiJW+nZ5zbqFDKcsnj9wljOjGf7Xd1/iwsXdrF4wh8//YAdP7D3NdRctQmSqm+cXLljAqnO6+MZT+zHGcN+Tr3Hfk/u56sIFdCcbl91f0ycZY7YDpU7wFuB+Y0wa2Ccie4DL3df2GGNedfe73932lVqOQ1EUpRY2vXiYiMDfffhy7n50D/f+dB/feHo/C7uTfOO/XsHCnmTZfa+8YAGxiPDozhN88+n9nB3Lct/tV7B+aS/bDg/y4AuHue0tq0ruG4kIH3jzSj77/e3cdu8z/GT3Ka5Zu5Av/PIb6nSmpamX5CwDnip6fshdAzg4af2KOh2DoijKjBhj2PTiEa68YAELe5L8n3etx7JtHtt1kn+4/QqW9nVOu39PR5yNq+bxjaf2M5yy+K3r17B+aS8AlyybyyXL5k67/y+/aTlfeHgnT+w9ze/dtI6PXHUekQYFiT1mFAIR2QycW+Klu4wxD5bbrcSaobQrqmQ7PhG5A7gDoL+/v9QmiqIoNfOzg2c5ODDOx651qnwjEeGz734dxpiS7pxSXLN2EU+9OsBFS3r571dfUNXn93UluPdDl9GdjM1YoVwvZhQCY8z1s3jfQ0BxZGU5cMR9XG598ufeA9wDsHHjxup6tyqKolTIpheOkIhFuPGSib93KxUBgHe9YSkPbTvG59/zurKxhOm48oIFVe/jJ/VyDW0CviUiXwKWAmuAZ3AshTUisho4jBNQfn+djkFRFGVarJzN935+lGvXLqK3Iz7r91nW18mDd17p45E1llrTR98jIoeAtwDfF5GHAYwxLwMP4ASB/w240xiTM8ZYwEeBh4HtwAPutoqiKGRzNk/uPd2wz3vq1QFOjaS55Y1LG/aZzUhNQmCM+VdjzHJjTNIYs9gYc2PRa58zxpxvjFlrjPlB0fpDxpgL3dc+V8vnK4rSWvzLc4d4318/xb5Tow35vE0vHqYnGeOadY0r3mpGtLJYUZSmYetrZwA4MDBW98/K2YYt209wzbpFUyqG2w0VAkVRmoafHXCE4GiJRmx+88LBM5wezTSs1XMzo0KgKCFjLGMxlrGCPgzfOTOa4VXXJXR0cPpunn6wefsJYhHhFxvYyqFZUSFQlJDxOw+8yFV/8iivnhwJ+lB85YWDZ/OPjw7W3yLYsv04l62az9zO2WcLtQoqBIoSIlLZHI/uPMGpkQwf+PrTHDpTf196o3j+wBmiEWHNou66WwQHTo+x6/iIuoVcVAgUJURsfW2AVNbmf964lpG0xa9+/WlODNXfjdIInj9whnXn9nD+wm6O1DlGsNkdL9nIVs/NjAqBooSIx3edJBGN8OErV/F3t1/OyeE0H7v/Z0EfVs3kbMOLBwe5tL+PJX0dHB1M5VtBl6LUa8YY3vtXT/LJ77zEuDtYvhxbdhzngkXdrDxnTs3H3gqoEChKiHh81yk2rppHVyLGhv55/MfXL21Yzn092X1imJG0xYb+eSyd28lYJsfQeOmA+JGz47zps5t5Ys+pCeunRzM8vW+Abz9zgHff/VP2nCjEUAbHs/lBMUOpLE+/OsD1F6lbyKNxDa8VRamJ40Mpdh4f5hMb1uXX4jHByoW/Fdfz+51A8Yb+eWw7MgjA0aFx5nZNDeQ+8OxBBkYzPLv/DG8t6tHj1R7cfuVqvvvCYW7+6r+zfkkvr54aZWA0w8KeJB966yrmdsYbOhg+DKgQKEpIeHyXMw7xqjWFdMd4NEImZ5fbJTQ8f+AM8+ckWHlOF6dH0wAcPZti3bm9E7bL2YYHtjqd7PefnhgoP+gKwfsuX8F/u2o1n3rwZc6OZ7nx4sX0z5/Dk6+e5gsP7wSc4fGX9jdmMHwYUCFQlJDw+O5TLOxJctGSnvxaPBppCYvgZwfOcOmKPkSEJXOd/v9HSqSQ/mT3SY4MpkhEIxwYmOgSO+AKw/J5XXQmotxz24TpufzG1eez49gQf//EftYv6WnYYPgwoEKgKCEgZxv+ffdJrlk3ceRhPCp533dYOTuWYe/JUd5zqTO7alFPkog4FsFk/nHrQebPSXDVmgX8dFJzugMDYyzqSdKZKN8uYt25vXz+l17n7wm0ACoEihICth0e5MxYdoJbCCAWiWDZpqohKs3A3Y/u4b4nX2NpXyc9bvvnDa6rJhaNsLi3Y4pFcGokzSOvHOdDb11FX1ec775whLGMRVfCuY0dGBijf35XQ8+jVdCsIUUJAV584G1rJg4w8YagZEPkHvqrx/byhYd3suqcOXTEouw6NsySuR0TpnMtmdvBsUlFZd95/hCWbXjvZSvyaZ/FcYKDKgSzRi0CRQkBT+w9zcVLe1nQPXGIesz1c1u2TSIEv+u+8dR+Pv+DHbzr9Uv4yq2XlvXTL+nr5JUjQ/nnxhju33qQN62cx5rFPYxnnTqB/afHuGhJL2krx9GhFCtUCGZF839zFKWNODGUKtlQbtfxYV5XYgh6POpaBFbzWwQPv3yM//3gNq6/aBFffu8bpw3WLunt4MjZ8Xzh2IuHBnn15Cjv3ehMul0537MInIDx4TPjGINaBLNEhUBRmgRjDDd/9af86cO7JqyfGc1wejTDBYu6p+wTjzo306zd/AHjbz9zgBXzuvjq+zfkBawcS/o6SVs2Z8ayAPx45wlE4O1ub6C5XXH6uuLsd1NGvRqC/nNUCGaDCoGiNAnHh9IcG0rx4qGzE9b3ul1Gz19YSgi8GEFzC0HONjz32hnetmZBRUNgls7tAMj3HPrJ7lO8ftlc5s1J5LdZOb8rnzLq1RCoRTA7VAgUpUnYcczxie86Njyhl44nBKUsgpgrBM1eS7Dj2BDDaYvLVlVWxLWkz6klODqYYnA8ywsHz3LVpLkBK8+Zw2uua+jAwBjJWISFk2IoSmWoEChKk7Dz2DAAw2mLw0XdN/ecGCEZi7DUvTkW47mGmr26eOu+AQAuWzW/ou09i+DY4DhP7DlFzjb8wprJQtDFkbPjZCw7nzoa0SKxWaFCoChNgicE4ASHPfaeHOW8hd0lg6vxKi2CoVSWr2zejdVg4di6/wxL53awfF5lrptzupPEIsKRwRSP7z5JdzLGpf19E7ZZec4cbAOHzoxxYGBc3UI1oEKgKE3CzuPD+ZvdjiJR2HNihPMXlm6XXG2M4NEdJ/jy5l28cnRo5o19whjD1n0DbKzQGgCIRoTFvR0cPTvO47tO8dbzz5kSYF7pBob3D4xxcGBMU0drQIVAUZoAK2ez+8QIl62az9K5HXnrIJXNcfDMWMlAMUDMyxqqUAiGxp0snLNuNk4jODAwxonhNJetrlwIAJb2dfDkq6c5fHZ8SnwACkLwwoGzjKQttQhqQIVAUZqA106PkbFsLlzcw4Xn9uSFYN+pUYwpHSgGSESrqywe9IRgvHFCsPW1MwBcXoVFALBkbifHh5xOpKUGzC/sTtKViPKT3U7VtQrB7FEhUJQmwIsJrDu3h7Xn9rD35AjZnD1t6igUVRZXaBF4QjA4lqn1kCtm674B5nbGWVNGzMqxpM8JGK9eMKek20dE6J/fxYuHnPkFWkMwe1QIFKUJ2HFsmIg4v/zXndtDNmfYd2qUPSdGEIHzysUI3F5DlWYNeVO/Guka2vraABtXzqs6o2ep2476qkn9lYpZeU4XOduxhlZUGIhWpqJCoChNwM5jQ6xaMIeOeJS1i51hLDuODbP35CjL53WWLcKKR6rLGmq0a+jUSJpXT41WHR8AWD7PFYISbiEPr/ncwhnaTyvTo0KgKE3AzmPDrF3sDJw5f9EcohFh17Fh9pwY4YIybiFwRlVC5cHiwQYHi599rbr6gWJ+8cKF/OUHNnDN2vIjJb2AscYHakOFQFECZjyTY//AGGvPdYQgGYuyesEcth8d4tWTI2XjA+DMIwDI2pXXEUBBEOrJoTNjfOuZgyRjkZIN82YiFo1w0yVLpnUpec3nVAhqQ9tQK0rA7D4xjDFOoNhj7bk9/Gj7CdKWXTZjCIqyhqwqg8Xj9QsW7zkxwle27Oahl44iwJ3XXJCfm+A3nkWgNQS1oUKgKAHjFY9duLggBOsW9/D9nx8F4PxphKDaOoJGuIb+8HuvsHXfAL/2ttV86K2rSrbG8Ivl8zr52LUX8O43Lq3bZ7QDKgSKEjA7jw3TEY/kA58AFxZZB9PGCKKVu4Zs2zCSdrOG6ugaGhrPctnq+fz+Oy+q22d4iAj/44a1df+cVkdjBIoSMLuOD7NmUc+EXkKem2j+nMSE1suTyc8jqMA1NJyyMAaSsQiDY9kJHU79ZCxjMUczeEKFCoGiBMyOY8MT3ELg5MR3JaLTWgNQ1HSugsE0nluof34XmZydH/foN6PpXH6gvBIO9GopSoAYYzg5nGbZvIl+9EhEuO0tq1g1Q7VsIUYw8697L2Oof34Xu0+McHYsW5cb9ljGYk5SLYIwoUKgKAHi3cAT0akpkp94x7oZ9/cKyioJFnsWgZdhc3YsW5dA7mhGLYKwoa4hRQkQ7wY+0wzfckQiQjQiVQmBl3J5tg4ppNmcTcayNUYQMmoSAhH5gojsEJGfi8i/ikhf0WufFJE9IrJTRG4sWr/JXdsjIp+o5fMVJex4rSFisxQCcALGlbSYGJokBIN1SCEdyzhxh66kWgRholaL4BHgEmPM64FdwCcBRGQ9cCtwMXAT8BciEhWRKHA38A5gPfA+d1tFaUu8ZnGlXEOVEo9EKmo6VxwshvqkkI5lnPRUtQjCRU1CYIz5oTHGcp8+BSx3H98C3G+MSRtj9gF7gMvdvz3GmFeNMRngfndbRWlLvGyfmiyCWKQii2BwPEs0Ivm4QD3aTIym1SIII37GCG4HfuA+XgYcLHrtkLtWbl1R2pK8a6iGoeuxCmMEQ6ksczvjdMajJGKRulQXq0UQTmaUbRHZDJxb4qW7jDEPutvcBVjAN73dSmxvKC08JX/KiMgdwB0A/f39Mx2mooSSvGuohl488WikovTRwXGL3o4YIkJfZ7wu/YbyFoFmDYWKGa+WMeb66V4XkQ8C7wKuM4VSxUPAiqLNlgNH3Mfl1id/7j3APQAbN26sTwmkogRMwSKoLVhcadbQ3M44AHM743W1CLrUIggVtWYN3QT8HnCzMWas6KVNwK0ikhSR1cAa4BlgK7BGRFaLSAInoLyplmNQlDBTSB+tIVgcjVRUWTw0nqXXFYK+rvoIwaibNaQFZeGiVvvtq0ASeEREAJ4yxvy6MeZlEXkAeAXHZXSnMSYHICIfBR4GosC9xpiXazwGRQkttdYRgBNozlhTjWZjDO7/l4AjBF4F89zOBIfPjs/6M8sxlvYsAnUNhYmarpYx5oJpXvsc8LkS6w8BD9XyuYrSKli2V0cwe4sgEZUpFsGDLxzmjx7azk9+99p8/MELFoNjEbxyZHDWn1mOvEWgQhAqtLJYUQLE6xpaq0UwOUaw79Qox4fSHDrjeGyNMRNiBH2d8frUEbgWgc4PDhcqBIoSIN4cgdpiBDIlayjtCsz+AUcIxrM5sjlDb0fBIhjL5Ehb/nYgHc3kSEQjdZtIptQHvVqKEiB+WATxEhZBxn3fA6cdIRgad36p57OGupwZB34XlY1lLLo0UBw6VAgUJUDylcU1pY9OrSz2fukfcC0C74Zf7BoC//sNjaZzGh8IISoEihIgnkunFtdQqcpizyLYf3qiEPR2Ojfpvi5HCPyOE4xlLK0hCCEqBIoSIH6kj8ZjU11DXozgwMAoUOg8WlxQBnWwCDI57TMUQlQIFCVACm2oa+k+WiJYnPWEYCyfMQTFriEnRuC7RZDWecVhRIVAUQKk0Ia61hjBJNeQ+zyVtTk5nJ4iBHM919CYv/2GdDpZOFEhUJQA8W7gtbShjkUjZMoEi8FJIfXmFfe46aM9yRgRqU/WkLaXCB8qBIoSIPWqLM5YNkvndgBOwHhwPEtPMkbUbXcdiUhdGs+NptUiCCMqBIoSIH64hmLRSL4ewSNt2axeOIeIOHGCwaKGcx59XYm6ZA1pjCB8qHQrSoD4MZgmHo3kK5Q9MpZNdzLGkrmdHDg9ykg6N0UIHIvAvxiBbRvGs5o1FEb0iilKgGRzNiLkXTazodQ8grRlk4hF6Z/fxf6BMeKRCHM7J/7v3tcVZ2DUPyFIWTmM0elkYURdQ4oSINmcIR6JTGgXXS3xaARjIFdkFWQsm2QswspzujjgxgjmlrQI/HMN6bzi8KJCoCgBYuXsmgLFUAg0F1sFaStHIhah/5wuTo9mODo4nm8459Hns2tI5xWHFxUCRQmQbM6uqaoYCoHmCUKQdSyC/vldAAylrKkWQVeCoZQ1wZKoBZ1XHF5UCBQlQLK2qanPEBQCzcXVxemcTTIWZeX8Ofm1yULgNZ4bTvnjHspbBFpHEDpUCBQlQLJW7RZB3O397xWnGWPIWHbeNeQxNX3Uqy72Rwi86WRqEYQPFQJFCRDLNjXHCOJuC2uvJsH7NxmLMLczPqXRnIffHUi96WRqEYQPFQJFCZBszs7fyGdLPOYIiVeT4HUeTbqWwkrXKpgsBN7cgJGUVdPne+i84vCiQqAoAeJHsNgbauMFizOThMALGPdOqiPwxklmcv6Mq/RiBDqPIHyoEChKgFg5H1xD+ayhiRZBYpIQTLYI8kIwqT3FbPGyhuZoHUHoUCFQlADJ+GARxCfVERQsAueX+ZtWzqMnGWNxb8eE/ZJ5i8Cf9NGxjEVECu+rhAeVbkUJECtXe/qoJyReB1KvBbV3Q77uosW88KkbprSxSEQdofDTIpiTiNVUJa0Eg0q3ogSIZds1Da6HQmVxxnJdQ9mJriEo3cvIb9fQWMaiSzOGQokKgaIESCZn8nUAsyUxySIopI9Of1MuCIE/wWKdThZeVAgUJUCsnE28hs6jUJhu5sUISlkEpfBcUpmcTxZB2tKMoZCiQqAoAeJH+mghWOy4hrx00JmCtr5nDWUsrSEIKSoEihIg/qaPVmcReC4l/7KGchojCCkqBIoSIFnbD4vA6zXkWQQTC8rKISIkohEfs4bUIggrKgSKEiBZy7/uo5kqLQJvG/+yhnIaIwgpKgSKEiCWbeeDvbMlEZtoEaQrzBry9vWrxcRo2tKq4pCiQqAoAZKx7LyvfrYU5hF4FoEbLI5XYBH45BoyxqhFEGJUCBQlQCzb5G/ks8WrQ8gLgddrqAKBicfEFyHI5Gws26hFEFJUCBQlQJysoRqDxfnuo26w2KosWAyuReBDHcFYfkylWgRhRIVAUQLCGEMmZ5OoOX3Um0dQsAgS0UhFPX8SsWi+NUUtjOYH16tFEEZUCBQlILyh8bVaBNHI1O6jlXYAdYLFPlgE3phKrSMIJSoEihIQniun1joCrx4ga3vzCHIVpY4CJKORKb2GjDF8+ZFdHB0cr/gYRtNqEYSZmr6BIvIZEfm5iLwgIj8UkaXuuojIn4vIHvf1DUX7fFBEdrt/H6z1BBQlrGTdJnG11hGA04E0a83SIpgULD46mOIrW3az+ZXjFX9+3iLQGEEoqdUi+IIx5vXGmDcC3wP+j7v+DmCN+3cH8DUAEZkPfAq4Argc+JSIzKvxGBQllHh5/7VmDYFjVVh2YUJZMl7ZDbmUa8jLOkpXkU2Utwg0ayiU1CQExpihoqdzAC/qdAtwn3F4CugTkSXAjcAjxpgBY8wZ4BHgplqOQVHCiufTr7UNNThCkCmKEVRamxCPTk0fTbl1CN6/laAWQbipWb5F5HPAbcAgcI27vAw4WLTZIXet3Hqp970Dx5qgv7+/1sNUlKYjLwQ1DqYB54ZeyBrKVVRMBl7WkA8WQUYtgjAz47dFRDaLyLYSf7cAGGPuMsasAL4JfNTbrcRbmWnWpy4ac48xZqMxZuPChQsrOxtFCRGeayge88c1VDy8vlKLIFG0n4dXmVyNEGgdQbiZUb6NMddX+F7fAr6PEwM4BKwoem05cMRdv3rS+o8rfH9FaSk8i6DWUZXgBouL00crtggiU274eYugCteQZxHohLJwUmvW0JqipzcDO9zHm4Db3OyhNwODxpijwMPADSIyzw0S3+CuKUrbUUgfrd0icH7ZTywoq4RkbGr66GxcQ2OZHB3xSMnZyErzU6t8/7GIrAVsYD/w6+76Q8A7gT3AGPBhAGPMgIh8BtjqbveHxpiBGo9BUUJJPkZQYx0BeBZBocVEJZ1HoXTW0GyCxTqLINzUdOWMMf+pzLoB7izz2r3AvbV8rqK0At6w+Vori8GLERSCxZUWlJXqPjobi2Bcp5OFGq0sVpSA8NM1FI9EZtViIh6NYJtCnyJwhMT5t7qsIbUIwosKgaIEhJ+uoXhMCoNpqgwWO8dSyBzyJpylrerqCDRjKLyoEChKQPhZWRybZBEkopXHCLx9PFKWFyOorrJYawjCiwqBogRExk+LYFIdQbUWQbpoXOVsLYKOCttaKM2HCoGiBITlU/dR5z2cOgLb9mYcVN59FCZaBIU6gsotglQ2R6cKQWhRIVCUgLB87D7qNZ3zrIxqLYKJQlB9sDiVtemo8DOV5kOvnKIEhHfz9auOIGPZVc0rhiIhyJWwCKpwDY2rRRBqVAgUJSCs/IQyfyqLLdsuzCuu8KYcL+EaKhSUVeca0hhBeFEhUJSAqEdlsfcrvprBNMXHAtVbBLZtqpqBoDQfKgSKEhD5gjJf2lA76aN5i6CKymKYGA8oZA3ZOE0CpsfbV11D4UWFQFECwsoPpvGrDXUhRlCtRVAqWGwMU1pUl8JzJWmwOLzolVOUgPCzDbUzmMbkb+gVD68vKQRTRWE6xl0hUIsgvKgQKEpA+NlrKBZx0ke9X+fVdB+FSVlDRV1HKwkYFywCFYKwokKgKAGRzdnEIoKID1lD7g3dGxBTTfdR8MciUNdQeNErpygBYdnGl9RRKPQrGklXlzUUL+Ma8rSpkqIyz2pQiyC8qBAoSkBkc7YvqaNQSEEdTTsWQcWuoWiJ9NFsjh63gVwlbSbS6hoKPSoEihIQ/gqB8xPeE4KKXUOxqemjKcumtzPurmuwuB1QIVCUgLByxpcW1FCwCEbyFkGVWUOTLILeDkcIKgsWq2so7KgQKEpAZHy0CGKTXEO1BovnVmERaB1B+NErpygBYeWML6mjUHANVRssjkSEWETyQmDlbCzbFAnBzBaBuobCjwqBogSEZdcvWFypReBt6wmB5yLq7XSDxRVlDbnio0IQWlQIFCUgMpbJu3RqZYoQVPG+XnsKKPj7CzECdQ21A3rlFCUgHIvApzqCvGvIIhGLVFWklohF8paAFxPorcI1lMraRKQ68VGaC71yihIQfqaPejfh0YxVcXygeN/J4ynzMYIKLYKOeNSXCmklGFQIFCUgsj6mj3rvM5rOVVxM5pEsihF4glBtsFgDxeFGhUBRAsLK2VUFdafDaxUxkp6FRTBBCBwLoDtZTbDY1hqCkKNCoCgB4adF4A23GZ2tEEwKFnfEoyRjkYpdQ0kNFIcavXqKEhDZnO1f1pA73GYsk6vaykg4NR6GAAAQbUlEQVREp1oEyXjEEYIK00fVNRRuVAgUJSCyOdu3TJvi4TbVWgTF6aNesLgjFqUjHq2sstjSwfVhR4VAUQLCzzbUxYJStUVQIlicjEdIxiMVdR8dz6hFEHZUCBQlIJwWE371GioISrVZQ4kiF1BhwlmEZCxaRbBYbyVhRq+eogSE03TO3+6jUL1raGJBmWsRxJxgcaWVxdpeItyoEChKQFg525fB9TBx7nG1rqFkqWBxTIPF7YQKgdJw/mzzLt7/108FfRiBk/XRNVSzRTApRtARryZYrK6hsKNXT2k4248Osf3oUNCHEThZH11DsRosggmuITc4nKjCItBgcfhRIVAazkjaYiRtYYwJ+lACxbJ9tAgmpI9Wd1OORyNkvWCxlSMeFaIRIRmLzhgjMMZo+mgLoEKgNJzhlEU2Zyr6tdmq2LYh52P6aCTi3LyhdovAE5JkfGaLIJOzMUbHVIYdX4RARD4uIkZEFrjPRUT+XET2iMjPRWRD0bYfFJHd7t8H/fh8JVwMp5ye+UOpbMBHEhxZ27nB+mUROO/lCMFsuo9mcwbbNqStXH5/p8XE9EKQyui84lag5m+hiKwA3g4cKFp+B7DG/bsD+Jq77XzgU8AVwOXAp0RkXq3HoISLYVcAPEFoR7I5xy3mV4wACu6h2VgE4Py6T1uFBnKVBItTlg6laQX8uHpfBn4XKHb43gLcZxyeAvpEZAlwI/CIMWbAGHMGeAS4yYdjUELEkCsA7SwEluuK8St9FAodSGfThhocIUhlJ1kEM7iGxjM6r7gVqOlbKCI3A4eNMS9OemkZcLDo+SF3rdy60iakrVw+VXGkjYUgbxH41IYaCjMJZpM+CpCxHIsgUSQoqWxu2qB+wSJQIQgzsZk2EJHNwLklXroL+H3ghlK7lVgz06yX+tw7cNxK9Pf3z3SYSkgovvkPt3OMwLUI4j61oYZCvGE23Ue9Y0pbdr5KOBmLYBsvu6n0cRbaVqtrKMzMKATGmOtLrYvI64DVwIvuiLrlwPMicjnOL/0VRZsvB46461dPWv9xmc+9B7gHYOPGje2dZ9hCDE8Qgva1CCzXIvCrDTXMPljsCUjGskkXu4bcm3vaKj9S03MNqUUQbmb9LTTGvGSMWWSMWWWMWYVzk99gjDkGbAJuc7OH3gwMGmOOAg8DN4jIPDdIfIO7prQJxTf/ds4a8tI1fQ0Wz9YimOQaKg4Ww/Rzi9U11BrMaBHMkoeAdwJ7gDHgwwDGmAER+Qyw1d3uD40xA3U6BqUJKXYHtbVFUIf0Uc+6mE33UXB++aeyORb2JN33KayXI6XB4pbANyFwrQLvsQHuLLPdvcC9fn2uEi6G04Wb/0i6jYUgnz7qnxAkZltHUJQ1lLHsoqwh5+Y+XXWxWgStgUZ4lIYyrMFioOAa8quy2HmvQtpnNSSjE11D+criSiwCDRa3BHr1lIbi3fwXdCfb2zXkWQR+1hF4FkGVN+WJMYJc/qaejxFMIwRaR9AaqBAoDcW7+S/t62hrIcjWM1gcnV2MIJuzJ/Ya8iwCdQ21PCoESkMZTmXpjEfp60q0tWsom3cN+WkRTEz7rHa/jGWTsnL5/YvTR8vhBYurdUcpzYVePaWhjKQtujti9HTEJgSO2w2rDr2GvMriRJXi4lkEKStHNmeqDBY7Q2ncWiIlpKgQKA1lKGXR0xGjJxlT1xA+dx+Nzc4i8ITDux7VBYt1FkEroEKgNJThlEVPR9yxCNrZNWTXo/vo7CwC74bvCUG1wWINFIcfFQKloQynsvR2xOjpiJPK2vlfxu2GNxHM33kEnkUwu2CxV+k91SKYyTWkQhB2VAiUhjLiuYY6Yvnn7YhXWexnsHi2dQR5IRj3XEOTYwTTu4Y0UBx+9AoqDWU4ZdGddCwC73k7km9D7WP30URUECkEjSvfz7kNeJXeU7OGprEIsjk6E2oRhB0VAqWhDKey9HTE6U46FkG7Np6rR7A4EYvQEYtWncETjTgCMjzJNeQJxHTjKlPZHB1V9jZSmo96NZ1TlCnkbMNoJkdPR4xe1zXUrhZBoQ21fxbB+y7v53XL+6reT0RIRCNTgsWRiJCYYUrZeDbHop747A5YaRpUCJSG4cUDnKwhzzXUphZBHbqPnrewm/MWds9q30QsMsUicB5HZnAN2dpnqAXQK6g0jOG0c6PpSRYFi9u0qCxr+d99tBaSsciUYLHzODpjsFhdQ+GnOb6FSlswnLcICkLQtq4h20bE8c83A45ryLUI4sVCMJNFkKNDg8WhR4VAaRjDRa6h7rwQtKdrKJMrP/4xCBKxCKP5vkFFrqH49DGCVNZWi6AFaJ5votLyeDf9no4YyVjU9Uu3qUWQM76mjtZK8XjLYp9/Ryw6bdbQeDZHZ0JvI2FHr6DSMIpdQwC9HTGG2lYI7HxvoGag2DqZahGUdg1lczY526hF0AI0zzdRaXm8bqOeW6inI962weJMzhDzcShNrSRiE+MCxY/LWQReV1JtMRF+muebqLQ8nmuo100dbefGc1bO9rXhXK0kouWEIFrWIhj3hECDxaFHhUBpGMMpi3hU8jea7jZuRZ1twmAxOO0pivsfdUwTLPYshY4mcnEps0OvoNIwvPYSXguEdrYIsrbxtaq4VgqN5iKT1qNlhcCzCLTXUPhRIVAahtdwzqOnI962FoGVs6ueG1BPEvmhNhNv6slYpOyEsnyMQIPFoad5volKy+O1oPbo6Yi1bRvqbK65LIJEmRbW09UReBXHGiwOPyoESsMYniwEyRgjGQvbndbVTmRzdlNlDcXLCUEsSrqMRVBwDTXPeSizQ6+g0jCG3BiBR09HHGNgJNN+VkG2SV1Dk3/dTxcs9lxDSXUNhZ7m+SYqLc9wyqInOdE15K23G1azuYamCRZbtsEqMVI0pcHilkGFQGkYTtbQxGCxt95uZG3TlOmjk3/dF+YWlxcCjRGEn+b5JiotjTGGkbQ1yTXUvnOLs1ZzFZQl84PvJ1sE0wmB1hG0CnoFlYYwlslhGyZYBN3t7BqymytYXNY15P7aL1VdrHUErUPzfBOVlqa4BbWHN66yHecWZ3OmqZrOlasj8DqRluo3pHUErUPzfBOVlsaLA3SXjBG0n0WQzdlN1YZ6uvRRKO8aSkQjRJroPJTZoUKgNASv8+jkgjJoz3GVVi5cweJS1cWpbE7nFbcIehWVhuD96u8tEoLOeJRoRNozayhnN1f66KwsgpxmDLUIKgRKQyhMJyvECESkbTuQNmv30VIFZVA+WKyB4tageb6JSkvj3eyLm86B14G0HYXANFf66DQFZVA+WKyB4tZAhUBpCMXziotxOpC2n2vIsu0Jff+DppA1NLXpHJQPFmuMoDWo6SqKyKdF5LCIvOD+vbPotU+KyB4R2SkiNxat3+Su7RGRT9Ty+Up4GElZiMCchFoExhjXImiem2gi6vyyryZYPK4xgpYhNvMmM/JlY8yfFi+IyHrgVuBiYCmwWUQudF++G3g7cAjYKiKbjDGv+HAcShMz5M4imJxq2NsR48jZVEBHFQyW2221udJHnWOpJliczubo60rU/+CUuuOHEJTiFuB+Y0wa2Ccie4DL3df2GGNeBRCR+91tVQhanOGUlZ9VXEx3MsZwur1cQ1bOEYJmdA1VGyxeohZBS+CHEHxURG4DngV+xxhzBlgGPFW0zSF3DeDgpPUrfDiGkpwdy/Arf/lkvd5eqYKjgymW9XVOWe/piHPkbIq3f+mxAI4qGHLGtQiaKljs3NATZSyCux/dw7eePjDhtddOj7J+SW9jDlCpKzMKgYhsBs4t8dJdwNeAzwDG/feLwO1AqW+4oXRMouRUEhG5A7gDoL+/f6bDLEkkIqxZ3D2rfRV/WbO4m2vWLpqy/p4NyxgYy2BMew2nuWTpXK5dN/W/R1BcuLibj/zieVy1ZsGE9c5ElN+4+nz2nx6dss+axd38ysYVjTpEpY6IX/8Disgq4HvGmEtE5JMAxpjPu689DHza3fTTxpgb3fUJ25Vj48aN5tlnn/XlOBVFUdoFEXnOGLNxpu1qzRpaUvT0PcA29/Em4FYRSYrIamAN8AywFVgjIqtFJIETUN5UyzEoiqIotVFrjOBPROSNOO6d14CPABhjXhaRB3CCwBZwpzEmByAiHwUeBqLAvcaYl2s8BkVRFKUGfHMN1RN1DSmKolRPQ1xDiqIoSvhRIVAURWlzVAgURVHaHBUCRVGUNkeFQFEUpc0JRdaQiJwE9tfwFguAUz4dTlhox3OG9jzvdjxnaM/zrvacVxpjFs60USiEoFZE5NlKUqhaiXY8Z2jP827Hc4b2PO96nbO6hhRFUdocFQJFUZQ2p12E4J6gDyAA2vGcoT3Pux3PGdrzvOtyzm0RI1AURVHK0y4WgaIoilKGlhYCEblJRHaKyB4R+UTQx1MvRGSFiDwqIttF5GUR+U13fb6IPCIiu91/5wV9rH4jIlER+ZmIfM99vlpEnnbP+R/dducthYj0icg/i8gO95q/pdWvtYj8tvvd3iYi3xaRjla81iJyr4icEJFtRWslr604/Ll7f/u5iGyY7ee2rBCISBS4G3gHsB54n4isD/ao6oaFMyb0IuDNwJ3uuX4C2GKMWQNscZ+3Gr8JbC96/n+BL7vnfAb4tUCOqr58Bfg3Y8w64A0459+y11pElgEfAzYaYy7BaWF/K615rf8OuGnSWrlr+w6cWS9rcKY5fm22H9qyQgBcDuwxxrxqjMkA9wO3BHxMdcEYc9QY87z7eBjnxrAM53z/3t3s74F3B3OE9UFElgP/Afi6+1yAa4F/djdpxXPuBa4C/gbAGJMxxpylxa81zuyUThGJAV3AUVrwWhtjHgcGJi2Xu7a3APcZh6eAvknDwiqmlYVgGXCw6Pkhd62lcUeGXgo8DSw2xhwFRyyA5hmS6w9/BvwuYLvPzwHOGmMs93krXvPzgJPA37ousa+LyBxa+FobYw4DfwocwBGAQeA5Wv9ae5S7tr7d41pZCKTEWkunSIlIN/AvwG8ZY4aCPp56IiLvAk4YY54rXi6xaatd8xiwAfiaMeZSYJQWcgOVwvWJ3wKsBpYCc3DcIpNptWs9E75931tZCA4BK4qeLweOBHQsdUdE4jgi8E1jzHfc5eOeqej+eyKo46sDVwI3i8hrOG6/a3EshD7XfQCtec0PAYeMMU+7z/8ZRxha+VpfD+wzxpw0xmSB7wBvpfWvtUe5a+vbPa6VhWArsMbNLEjgBJc2BXxMdcH1jf8NsN0Y86WilzYBH3QffxB4sNHHVi+MMZ80xiw3xqzCubY/Msb8KvAo8MvuZi11zgDGmGPAQRFZ6y5dhzMbvGWvNY5L6M0i0uV+171zbulrXUS5a7sJuM3NHnozMOi5kKrGGNOyf8A7gV3AXuCuoI+njuf5NhyT8OfAC+7fO3F85luA3e6/84M+1jqd/9XA99zH5wHPAHuAfwKSQR9fHc73jcCz7vX+LjCv1a818AfADmAb8A9AshWvNfBtnDhIFucX/6+Vu7Y4rqG73fvbSzhZVbP6XK0sVhRFaXNa2TWkKIqiVIAKgaIoSpujQqAoitLmqBAoiqK0OSoEiqIobY4KgaIoSpujQqAoitLmqBAoiqK0Of8fINrysGpj5WMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "actor_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.0005}\n",
    "critic_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.0005}\n",
    "agent = QActorCriticAgent(env, actor_kwargs=actor_kwargs,\n",
    "        critic_kwargs=critic_kwargs)\n",
    "\n",
    "# 训练\n",
    "episodes = 100\n",
    "episode_rewards = []\n",
    "for episode in range(episodes):\n",
    "    episode_reward = play_sarsa(env, agent, train=True)\n",
    "    episode_rewards.append(episode_reward)\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "# 测试\n",
    "episode_rewards = [play_sarsa(env, agent) for _ in range(100)]\n",
    "print('平均回合奖励 = {} / {} = {}'.format(sum(episode_rewards),\n",
    "        len(episode_rewards), np.mean(episode_rewards)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "优势执行者/评论者算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PlLt4F20zmUW"
   },
   "outputs": [],
   "source": [
    "class AdvantageActorCriticAgent(QActorCriticAgent):\n",
    "    def __init__(self, env, actor_kwargs, critic_kwargs, gamma=0.99):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = gamma\n",
    "        self.discount = 1.\n",
    "\n",
    "        self.actor_net = self.build_network(output_size=self.action_n,\n",
    "                output_activation=tf.nn.softmax,\n",
    "                loss=tf.losses.categorical_crossentropy,\n",
    "                **actor_kwargs)\n",
    "        self.critic_net = self.build_network(output_size=1,\n",
    "                **critic_kwargs)\n",
    "    \n",
    "    def learn(self, observation, action, reward, next_observation, done):\n",
    "        x = observation[np.newaxis]\n",
    "        u = reward + (1. - done) * self.gamma * \\\n",
    "                self.critic_net.predict(next_observation[np.newaxis])\n",
    "        td_error = u - self.critic_net.predict(x)\n",
    "        \n",
    "        # 训练执行者网络\n",
    "        x_tensor = tf.convert_to_tensor(observation[np.newaxis],\n",
    "                dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            pi_tensor = self.actor_net(x_tensor)[0, action]\n",
    "            logpi_tensor = tf.math.log(tf.clip_by_value(pi_tensor,\n",
    "                    1e-6, 1.))\n",
    "            loss_tensor = -self.discount * td_error * logpi_tensor\n",
    "        grad_tensors = tape.gradient(loss_tensor, self.actor_net.variables)\n",
    "        self.actor_net.optimizer.apply_gradients(zip(\n",
    "                grad_tensors, self.actor_net.variables)) # 更新执行者网络\n",
    "        \n",
    "        # 训练评论者网络\n",
    "        self.critic_net.fit(x, u, verbose=0) # 更新评论者网络\n",
    "        \n",
    "        if done:\n",
    "            self.discount = 1. # 为下一回合初始化累积折扣\n",
    "        else:\n",
    "            self.discount *= self.gamma # 进一步累积折扣"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0UYhQoYlzmUe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均回合奖励 = -10298.0 / 100 = -102.98\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd4XOWZ8P/vMzMa9V5tS5Zc5I4LNmA6BAMGEiCFlE0hlRTySzbZ95dN203fbDZ1eZewSxJCCglh03CAUAKhGmxccMVFtmVbzep1NJr2vH+cc0Yz0qjOjEajuT/XpcvSmaOZMzrjc5/nvp+itNYIIYRIXbZEH4AQQojEkkAghBApTgKBEEKkOAkEQgiR4iQQCCFEipNAIIQQKU4CgRBCpDgJBEIIkeIkEAghRIpzJPoAJqOkpETX1NQk+jCEECKp7N69u11rXTrRfkkRCGpqati1a1eiD0MIIZKKUur0ZPaT1JAQQqQ4CQRCCJHiJBAIIUSKk0AghBApTgKBEEKkOAkEQgiR4iQQCCFEipNAIESKcnv9/PqV07i9/kQfikgwCQRCpKhfbK/ny38+yB/3NCb6UESCSSAQIgW5vX5+8sIpAH6/+2yCj0YkmgQCIVLQQ7vO0t4/xJaVZew5082Jtv5EH5JIIAkEQqQYrz/A/zx3ko3Vhfzbm8/DpuAPuxsSfVgigSQQCDEDet1ebvzPF9jf0J3oQ+FPextp7B7kk1cvpSwvgyuXlfKnvY34AzrRhyaAV+s7OdDQM6OvKYFAiBlwuKmXw829PHu0LaHH4Q9o7nn2BKvn53HVcmN24rdtrKK5x832E+0JPbbpcHl8XPztp7n+h8/zncePsKu+E62TI6DdcvdL/OT5k2HbAgHNJx7Yw0d/tQuPLzBjxyKBQIgZcLpjAIBj5/oSehxPHmrhVPsAd169FKUUANesLCM/M43fxzk95PMH6Bn0Tut3Bz1+/u2x1+l2ecK2H2rqpbnHTUBr7n3+JG/775f5z6ePx+JwY6atb2hUa2vI52ff2W5+veN0WODac6aLtr4hmnrc/HnvzPXmkkAgRJR8/gCHmsZvytd3uAA4fi5+RdntJ9r5zuNHxt3nNzvPMD8/g+tXVwS3ZaTZedO6eTx+sIVe9/Qu1BNp7XXzpv96iZvuemFad+wvHG/j3udP8sj+5rDt+80UygMfvog9/3ItVy8v5b4XTzEw5IvJcUfrbKeLS//9GR7Z3xS2vaPfCGinO1wcbOwNbn/8YAtOu43l5bnc89yJGUvXSSAQIkq/393ATXe9yAM7xl4DxGoRnGzvx+ufXJO/rrWfs52uSR/HL7bXc8+zJ+ga8ER8/GynixeOt/P2C6qw21TYY2/fVMWQL8CX/3Qw5hefutZ+3vzj7bze3EtD1yANXYNhjw96/Lxa3znucxxoNC74e850hW3f39BNRV4GZXkZ5Gem8ck31NLr9vG/u2a2S2xLj5v3/HTHqPP1+90NePwBTrSG3wC09Q0Fv3/kgBEktNY8fqiFS5cW85lraznVPsCjB8IDX7xIIBBxc6ChhyMtvRPvGEdaa545co7WXnfcXuPpI60A/OvDh3jxeOQ8e327C6XA69fBoDCRD97/Ktf+8Dke2nV2wrtorTV7zhiF6P2NkVsnv3v1LDZlXPRHWltZwOe2Lmfbvia+8Mf9BGIQDPqHfGzb18Tb/ns7Qz4///bm84zjG1EI/fn2U7z9f14e9xxZv7PndHggONDQw3mV+cGfN1YXcv7CAu57qX5Gi99/2NPAi3Xt3PPcieC2QEDzhz1Guq015MIPw4GgLDedR/c3o7XmUJMRKLeuqeC6VRXUluXw47/XxeRcTEQCgYibTz24l688fChhr+8PaL72l8N88P5d/PsEKZPp8vgCbK9r5+Z181lamsMnHtg9qk++1sbFf+PCQgCOTSI91DPo5UynC6fdxud+v5/PPrSP/nHSHQ1dg8GLy4EIPZN8/gAP7TrLVcvLmF+QGfE5PnHVUj51TS0P7Wrgq385NKUUjtvr51BTD4/sb+Kup4/z3p/tYMPXn+RTv91LUZaTP3z8Et62sRKn3Taq59SOk51oDUdaItdPtNYcaOzBYVPUd7jo6DfeZ6/by8n2AdYuyA/b/yOXL+ZMp4snD7VM+vij9Zd9xl39H3Y3BI9vZ31nsPVzbkSQazP3ec/mahq6Btnf0MMTh1qwKdiyshybTfGJq5dwpKUveKMRTxIIRFy09Lg51T7AqfbJ3f2CcQfVOUZaY6rcXj+f/M0e7t9eT3G2k+ePtU/rzsrrD4ybN999uosBj5+b1s7jp7dvIs1u40P3vxp20W7v9zDg8XPNynKUmlzB2NrnB29fz2e2LOPh1xq584E9Y+5vpUzSHbZRd9wAzxxppbVviHdeMLo1EOozW2q544rF/PLl02zb1zTmfv6Apr59gAd2nOZD97/K+q8/yU13vcgnf7OXHzx1jKbuQT5w6SIevGMzT3zmCqqLs3E6bKyclxt2fP6ADh77WH+Xhq5BOgc83LR2HgB7zZbPQbPls7aqIGz/61ZXsLAoi5+8MNwjp6N/iEFPfOZUqmvt50hLH+++aCFDvgC/fuUMYKSFctIdXLy4eFSLoN38+V0XLiTNrnhkfxOPH2zhwkVFFOekA/CmtfNZWJTFf/29Lu49oaJavF4pdRvwVWAlcKHWelfIY18APgT4gU9prZ8wt28F/hOwAz/VWv97NMcgRnuprp0DjT187MolCTuGHac6AKNJ7PL4yHJO/FH7zhNH+M0rZ9j1L1tId9in/do+f4Db79vJjlOdfPmmlRRmOfmn/93H4eZe1oy4e5zIXU8f5497Gnn+c1ePyqsDPH+8DYdNccmSYnIz0vjubWv54P272F7XznVmQdZKBa2Yl8vCoqxJBYIjzUZKbdX8PLasKsdhV3z3iaMcauph9fzR72H36S6ynHauXlE2Kn0C8OCrZynLTecNK8rGfV2lFF+4YQV/3tvIU4fPccv6BWGPf/0vh3n+eBunOwbw+o2LU2VhJu+8YCGbagpZXJJDTUnWmOf7vMp8Ht7bRCCgsdkUx8710ec2guZYLQKrPvDui6p5dH8zu890sWVVeTCgnDfinNptig9eWsNX/3KYf334IK+d7WZ/Qw/nLcjndx/dPKnP4lQ8dqAZpeD/e0MtTd2D/OqVet57cTWPHWjm5nXz0RqeORp+V9/WP0RBVhqluelcXlvKg6+epc/t4z2bVwf3cdhtfO3m1Tgd8b9fj/YVDgJvAZ4P3aiUWgW8E1gNbAV+rJSyK6XswN3ADcAq4F3mviKGfvfqWX741LGE9qd+5WRH8PvTHRMXPOvbB7jvxVP0DfloHFFMnKrfvnqWHac6+Y+3ruXDly/m8mUlADx3bOp9+Ju63TR2D7JvjIFgzx1tY2N1IbkZaQBcsqQEu02F7W/1GKopzqa2LHdSqaEjLX3kZjiYl58BwHsuqibLaedn5vxAI+0508W6ygI2VBXQ3OOmtW84FdHUPcizR1t5+6YqHPaJ/8srpbhsaQnbT3SEtaLOdLi476VT5GU4+NBli/nOW8/jyc9cwQufu5qv3ryaN66dz6r5eeNeaNdWFtA35KPeDI67zKBVUzx2gNzf0EOaXbGuKp/V8/OCge5AQw+VhZkUZTtH/c5tm6oozErjV6+cxmFTvP+SGg419fDZ3+2Lec790f3NXFBdREV+Bh+5fDHt/R7ufGAPLo+ft22spDwvnfb+IXwhnQTa+oYoNe/8bzpvXjAYXre6POy5r15RxqVLS4JdfeMlqkCgtX5da300wkO3AA9qrYe01qeAOuBC86tOa31Sa+0BHjT3FTHU2udmyBegvT82aZbpeOVkJwvMXPRkiqP//tcjwTvMM1PoKTNSr9vLD586xkWLirhtUyUAZbkZrJ6fN61A4PYZ6YS/R8jTtva6Odzcy5XmwCwwumKuqAhPf5zpGMBuUywoyGRZeQ717QMTDhY62tLHyoq84AUgPyuNt2+qYtu+Jlp6wvPNLo+P15v72FhdyNpKI00SOjL1j3saCGh4xwRpoVCX1ZbQOeDhcPNwsf+p188B8KN3bODzN6zgHRcsZFl57pQuUmvNwq7199lV30lpbjpvWFHOsXN9ES/SBxq7WVGRR7rDzoaFhexv6MHrD7C/sZt1lQWj9gfITnfw+D9ewa4vbeGPn7iUr968mi/euJLHD7Xw/aciXbKm5/i5Po6e6wumrS5eUsyqeXm8fLKDmuIsNlYXUpqXgdbQEZL2bOsbojTXCATXri7HabexvqqAefmR6zfxFq82xwIgtP9Wg7ltrO2jKKXuUErtUkrtamtL7GjMZGPlIxu7o7uzni6rPvC2jcaFuH6CFsHOU508fqiF92xeCDCqC96+s92s+9qTbPnBc3zkl7v4wZNHx5xD/+6/19Hl8vAvb1wVdoG6Ylkpe0530TfFfvJuM6/8TIRA8LzZQ+iK2tKw7euqCth3tjt4UavvcLGgIBOnw8ay8lx8AT1u7URrzdGWPpZX5IZt/9Bliwhozf3b68O27zvbgz+gOb+6gNXz87Cp4Qut1po/v9bEhTVFVBVlTfp9X7bUaEW9ENIL6m+Hz7G8PJeFxZN/npGWluaQkWYLCQRdXFBTyIqKXNzeAGe7ws99IKDZ39ATDCAbqwsZ9Pp5+UQHZzsHw3oMjVSelxHMt4Px93vXhQu5++8n+O3OM1M67rF6ID2y30gL3bDGSAMqpfjIFYsAeOv5lSilKDcv+K29w3WCtv7hQJCXkcZ/vG0tX75p5ZSOKZYmDARKqb8ppQ5G+BrvTj7SLYIeZ/vojVrfq7XepLXeVFpaGmkXMYY28wPX0DX9O+toWPWBa1eVU5ztHLdFEAhovvnoYSryMvjijStJd9hGtQh2nOqgZ9BLTXE2x871cdczdaP6k4MRQH7+Yj1v3rBgVC3gymWl+AKa7Sc6Rv3eeAbNgHOoqXdUz4/njrVRkpPOqnl5YdvXVebT6x5Of5zuGKDavHjWlucA4YXRnkFv2IWmsXuQviHfqEBQVZTF1jUV/GbH6bABU9bfYkNVIdnpDpaW5QTz6oeaeqlr7eeWDfOn9L7L8jJYXp7Li3XGTVi3y8PO+k62rBq/xjARh93G6vn5HGjsprlnkMbuQTZWF7HMfK8j6wSnO130uX3BQHB+tdHz6hdmMBzZY2g8Sim+fstqLq8t4Qt/PMAX/3RgUovy/HbnGZZ/+a+8896X+cnzJzna0ofXH0BrzaMHmrmwpoiyvIzg/m9aO59v3rqG919aAxB8LPTzE5oaArh1wwI21RRN+r3E2oSBQGu9RWu9JsLXw+P8WgMQ2g6tBJrG2S5iZNDjp8+8SEw31z4y9TBVr5zsIDfDwcp5eVQXZ1HfPnZA2ravif0NPXxu63KynA6qirJGBYKTbQOU5Dj56e2b+PG7zwegd3B0V8rvPH4Emw3+/+uXj3rs/IWF5KQ7ppwecnv9lJl3bqHpIX9A88LxNq5YVoJtRBF5ndmLxaoT1He4qCnOBmBJaQ42ZaQUwLg4XPadZ7j773XB3z/SbDy2cl54IAD40GWL6XX7eChkwNSe010sLs2m0MyVn7eggP0N3Wit2baviTS74sY186b0vsFID71a34Xb6+fZo234A5otK8sn/sUJrK3M52BjLztPGYPILqgppLbMDJAjAoHV1fS8BcbfdH5+BhV5GcHi65pxWgSRpNlt3Pf+C/jolYv5zY4z3Hr3SxNOwf3AjtOU5qbT7fLyrcde5/ofPc+Kf3mcq773LHWt/bxxXXiQddhtvGdzdbBuZH1+rJb6wJAPl8cfbBHMBvFKDW0D3qmUSldKLQJqgZ3Aq0CtUmqRUsqJUVDeFqdjSEmhRcLppIYefq2Rzd9+etyugxN55WQnFy0qwm5TVBdnj5nz11rzP8+fZHl5LreavVMWFmVxtjP8uE+2DbC4xLhQ5Jn/uUameNr7h3hkfzMfuHRRxDyr02HjkiXFPHe0bUpF9EFvgLWV+czPzwhLDx1o7KHb5eXKZaNbq7VluWQ57ew720O3y0PPoDfYIshIs5stG+Pi8/0nj9Ln9vHHPQ3B4zpqBoll5aMDwcbqQjZWF/KfTx/naEufOZCsKzhGAYwLbXu/h4auQba91sSVy0qDQWIqLltagscXYOepTp56/Ryluelj5uSnYm1lPoNePw/uPEtmmp2V8/LITndQVZQZfO+WAw09pDtsLDNbUkopzq8uQGtYXJId/DxMRZrdxhduWMnPP3AB53rdfPZ3r425b11rPwcbe/nw5Yt5/B+v4MV/vprv37aOj1+5hDXz87m8toQ3rR0/yFoXfKtFYI33mDOBQCn1ZqVUA3Ax8KhS6gkArfUh4CHgMPA4cKfW2q+19gGfBJ4AXgceMvcVMRLaX3nkUP6J9Ax6+cYjrwPwf58+Pq3eFed6jfrA5sXFAFQXZ9HUMxixCb77dBevN/dy+yU1wbtqIxC4wi7WJ9r6WVxq3FHnZhg9UkYOrrLmblk9PzxNE+rK5aU0dg9yom3yYxuGvH4y0oxumS/WtTPk8+PzB/j+k0dx2m1cXjs6ENhtijUL8nntbHewPlJttgjASA8da+3jUFMP/7u7gZriLOo7XBxqMgqzR1r6qCzMDN5RjvSDt68j3WHj3T/dwTNHWulyeYMpEyCYN//Zi6do6XWP6gI6WRctLiLNrnjmSCvPHW1jy8qyUa2f6bDu7l8+2cGGhQWkmT2Zlpfnjeo5tL+xh9Xz88J6O51vBr3x6gOTcfXyMt56fiVHWiIXqcG4MbIpghf7ysIs3rqxkv9z/XLufvf5/OpDF1GQNX6QTbPbKM52Bv9vWoPJ5kwg0Fr/SWtdqbVO11qXa62vD3nsW1rrJVrr5Vrrv4Zsf0xrvcx87FvRvL4YzSpIVRVlTjk19P0nj9I5MMQdVyzmeGs/Tx6e+shMq9uoFQhqirPROnK94hcvnyY3w8GtIfnrysJM+oZ8dLuMO/4el5eOAU8wEGSnG4HA6m5nsVoIY108YbioO5X00KDXT2aanTesKMPl8bPzVCdf+8thXjjezjdvXROx6yLA+qoCDjf3BlNANSEF1mXluZzucPG1bYcpyEzjFx+8ELtNBeeVOdLcy4qK0a0BS3VxNg98eDNaa+741W7AaClYVs3Lw2FT/PqV02Q77dNO52Q5HZy/sJDf7jxD/5CPa1dFnxYC404+22mME9kUctzLK3I42Tbco8of0Bxs7An2hLJY73Xk+IHpqCnJZsgXoDnC9BZaax5+rYlLlpSE1QCmozQ3nba+8BZBSc4cCQRi9rFSQ+cvLKSxe3DSaZD9Dd386pXTvHdzNf+8dQU1xdMb0fjKyc5gfQAIpkRG1glae9389UAzt22sCut3vtDs2WKlk060GykUKzWUZreRmWYflRqyAkNexth92KuKsjhvQT73PHsiLIU2nkGzRXDJkhKcDhv/+vAhfvXKaT56xWLePk53zHWVBXh8AZ44dA6lCOuxU1ueiz+g2VnfyaevqaW6OJtLl5bw6P5mhnx+TrYPjCoUj7S0LIdff/gicjMc5GU4WFqaE3wsI80e7J10/eoKMp3TH5x3eW0JQ74AmebfIBZsZosJCCuQWsd80jznJ9r6cXn8oy7466sK+Oata7gtwpxJU7W4xLjBqI/Qi2vv2W7OdLq4Zf3UCu2RlOdlcM68SZtzqSEx+7T2DeGwKdbMz6d/yDfh/O9aa1r73Hz5zwcpyUnnn65fjt2m+PhVSzjY2Dvl4uqr9Z1cUFMUHIVrFUnrR/Qc+u3Os/gCmvdeXB223eqaaAWCk2YaZ0nZ8IUuJ8MxKjXUO4kWAcD3bltH/5CXz/zutUlNSub2+sl02sl02rl4cTGn2ge4dlU5n9u6YtzfW1dlXLyeO9bKvLwMMtKGL8ZWvntxSTbv3my8/zeeN48znS4e3tuEP6BZUTF2isuycl4eD995Kb/80EWjUjZWL5tbNkwvLWS5zGxFXV5bEvYeonV+dSFpdsWGhcN3+1bwO2oWjO99/iQOm+KixeG9aZRSvGdzNfmZU68PjFRjBoJI3Xkf3tuI02Fj65qKUY9NVVluevDmo61vCLtNUThBSmkmSSCYY1p7jf7JVUVGwXSsOsGxc33cevdLrP3qk1z4rafZ39DDl29aGSy+vXlDJfPzM/ivZybfKnB7/Zxs62dNSJ6+ICuNvAxH2Ohirz/AAztOc+WyUhaVZIc9R1XhyEDQT5pdUVU4XADOzXDQ6x4ZCCZuEYBxsfn6zWt4qa4jrKdOJFpr3N5A8AL40SsWc+v6+fzoHesjTjcRakFBJiU5Trx+HVYfAKPn0NbVFfzbW84L5sevW12Ow6aCi6qMlxoKVV2czfqq0QXcN29YwBvXzuPSJcWTep6xnLcgnzetm88HLl0U1fOM9PGrlvDHj18aFrgXl+TgMKed2Hmqk9/vbuAjVyymsnD64xYmUpGXQUaabVQg8PoDPLK/mS0ryya8uZiM8rwM2vs9+AOatr4hirOdE36GZlJsJ90QCdfa56YsN50FBcZ/nsbuwYjz6zxxsIXXznZz+8XVLCrJZs2C/LBmutNh46NXLuEr2w7x8smOSaUFTrYNENBG6sOilKKmJDusRfDEoRZa+4b49luqRz1HdrqDkhxnsKZwsm2AhUVZYcXC3Iw0+qdRI7DctqmSl0928KO/HWNTTeGY723IzFVnpBmvfcnSEi5ZOrn0iFKKtZUFPHOklZqS8AtZmt3Gf793Y9i2giwnl9WW8OzRNpx2W/BOdbouWlzMRYujCwJgFL7/77s2RP08I+VlpI0q9jodNhaXZnOwsZenDp9jQUEmn3pDbcxfO5TNpqgpzh6VGnqprp2OAc+0C+0jleWl4w9oOgaGwgaTzRbSIphjjKHrGSwoHL9FcLi5l+riLL52yxref+miiINZ3nFBFfPyM/j2Y0cm1YPoeGvkbo/VxdlhLYKfv1RPVVEmVy2PPDipsnB4LIHRYygn7PHcdEfEGkGaXQUv2uNRSvHNW9dQU5LNPz20b8zZRa3ZKjOnmRKxulqObBGM5Y1rjVz00rKcYEsh1Swrz+W5Y20cO9fP125eHVV9Y7IWlWSPahH87fVz5KQ7gus6R6ssZHRx6PQSs0VqftrmsNa+Icry0inMSiPLaR+z59Dh5t5RI2JHykiz87mtyznQ2MOfX5t4/dRj5/pw2NSodE9NcRYNXS48vgB7znSx+3QXH7hk0ZhN44XmoDJ/QHO6wxXsMWTJzXBE7DWUm5E26XlvstMd/PDt62ntG+KbjxyOuI81z9B0c+Przfx3zSQDwbWrykmzK1ZEGEiWKpabNxFbVpazJUa9lCZSU2KMdQmdFG7vmW7WVxVENQtuKKvXUWufm/b+8FHFs4EEgjnE6w/QOeChLDcdpYxJzhq7R3fb7HN7Od3hmjAQANyybgFrK/P57hNHJ5zP/di5fmpKskdNm1tdnE1AG2mqn75wktwMx7g9bhYWZdHU7aa+YwCPP8CSkvAWQU56hGLxoC84xmCy1lUV8LErF/PQrgaeOXJu1OPRtgguX1rCj96xnmtWTm5ahvzMNH7+/gv5zJZl03q9ueCKZaWsnJfHV2+euUmJF5Vk4wvoYOvZ5fFxpKUvYu1lusqCg8qGjEAgLQIRL+391vJ3xt3HgsLMiKkhq1fGqnEGX1lsNsWXb1pFc4+bn4Ys9BHJ8XN9wR4xoaw+9C8eb+Pxgy38w0ULyUkf+6K9sCgLf0DzUp0x4dmSspEtgrQxWgRTL3l96ppaVlTk8vk/HKDHFZ4isuYZmky6KRKbTXHrhgVTSvNcVlsypcnh5pp1VQX89dOXx7VAPJLVgj1l1rEONBiT+IX2aIqWdeE/dq4Pr19LIBDxYw0ms+4+KgszI04zcThk0ZPJuHBREVtXV3DPcyfGXFfW7fVzutNFbdnotIaVI//BU8ewKWNu+PFYF8JnjxpdVxePaBHkmt1HQ7t/9rl905puIN1h53u3raNzwMN3nghfztLttYrF8c9Ti8RZNGIswWtnjfmNYtkiSHfYKcxK41Cj8X9PAoGIG2sIe1me8SFbUJBFt8s7Ko1yuKmXwqw0KqYwWvLzN6zA6w/w/SePRXy8rrUfrSPPj1OS4yTbaafL5eVN6+ZPOOe61fV1+4l2CrPSRs2TY935D3iG31efe+qpIcuaBflcsrQkuPShxZoWY7qpIZEcirOd5KY7ggXjvWe6WViUFTaFdSyU5WYEb8KkRiDixhqwEpoagtGzkB5u7mXV/LwpLShSU5LN+y6u4aHdZznc1Dvq8eEeQ6NTQ0qpYKvgw5dP3B99Xn4mDpvC7Q2M6jEEw4EgND3UaxaLpys3wxE2tTMMBwJpEcxtSikWlQ73HNp7tiumaSFLWV568KZMWgQiblp7h1DKuAMHIzUEhBWMff4AR1r6JlUoHulTb6glPzONf3vs9VGDzI62GAO/xur/vmVVOW/ZsCDiersj2W0qeOyLIzxfTrpxwQ8dSxBNiwAg22nHNaIYbtUIZqILo0ismmIjEDT3DHKud4gNMUwLWawbNJBAIOKo1RyxaA2+qiwY3SI4aS6TONn6QKj8rDQ+fU0tL9a1B/P3luPn+lhUkj1mYfSz1y7jB+9YP+nXsuoEoVNLWIZbBEZx1x/Q9A9Nr0ZgyXJGahGYNYIYdSEUs9eikmwauwfZcdJYI2F9yLTesWKlbDPSbON2lkgECQRzSFufm9KQu46SnHScdltYz6HXzRzlymm0CADes7maxSXZfPPRw3hD+l0fa+0LG1EcLSsQRGoRjEwNWc3tqFoE6XYGPP6wlk6w15BT/pvMdYtKjFlyH37NmF9oOi3miVhLVpaa3btnE/mEJ4GdpzpHreMbSWvfULDHEBjdFxcUZtIQ0nPocFMvTruNJRFy75ORZrfx+RtWcKJtgN/sMNZ9dXl8nO0cZFmEHkPTZc1COm6NwAwAvebEetG2CPwBHZxWAobXK5Yawdxn9Rx6/ng7a+bnjRoLEwvWoLLZVigGCQSz3rleN+/52Q6++Wjk0a+hWnvDAwEYk5+FtggON/eyrCK6KQyuXVXOZUtL+N6TR2ntc1PXakwbHKlQPF1v2bCAL97v+bq3AAAazElEQVS4giWlkVoE4auUBaegzoyuRgCE1Qmk11DqsGpb/oBmfVXs00Iw3K17ttUHQALBrPeT50/i8QV4+UTHuNMmBwKa9v6hYB7SsqAgkzMdA7i9RtrjcNPEU0tMxFoEfMgb4NuPHQkuuxjL1FBZXgZ3XLEkYhM6Z8TiNFOZcG4sWeZzhtYJBr1+HDaVsvP+pJL8zDSKzW7K8egxBMYMpCCBQExR54CHB3acoTQ3nV63L2K3zeC+Lg++gA7rmQDG8oxdLi+3/NdLbD/RQceAJyb5z8WlOXzsysX8aW8jv915xpgxs3hmRoNmOe3YbSrYa8iagjqaGoEVXMJbBAFJC6UQq1UQr0BQmpuO02ELzgw8m0ggmMV+/tIpBr1+fmT2ttl+oj3s8dZed3Ck78hRxZYbz5vH/R+4gPb+Id790x0ArJpEF87J+MTVS1lYlMXu010sLs0Omyo6npRS5ITMQBqTFoGZGgodpGatTiZSw6p5eSwoyGRBwfgDHqcrI83Ow3deyu2XjJ5+PdEkEMxSvW4v92+vZ+vqCi5dWsLSshxeOtERfFxrzXt+toMb73rBCAjWYLK80c3Oq5aX8dinL2fz4iJzGcnYpHAy0ux87ZbVQGzTQpORk+4IFosns0zlRKy1kF1D4TWC6c4zJJLP529YwZ8+cUlce/SsnJcXtjTrbDH7jkgA8KuXT9Pn9nHn1UsBuHRJMQ/tasDjC+B02Nh1uiuYm//Ug3u51VxAY2RqyFKel8FvPrwZl9cf0z7MVy8v42s3r47pvCyTEToVdbxaBG5z4XqRGrLTHcEbglQjtzuz1C+213PFstLgKk4XLylh0OtnX4MxIdZvd54hJ93B129ZzSsnO7nLXOJwvEKUzabiMpDl9ktqWDfDgSAvIy0YAHrdPtIdtqi6/GU7IxeLZVSxSAUSCGahQEDT2hc+zH3z4iKUMpbQ63F5eXR/Mzevn8/7Lq7hredX0tTjJi/DkTI57dAF7PuinGcIICvdahGMSA3JqGKRAlKzHTTLWYOaQu9GC7KcrJmfz/YTHRRmORnyBfiHCxcC8I1bV7OvoTul8tm5GQ5OtA33GopmDAEMtwhcYS2CAPmZ0S9cLsRsJ4FgFnKZeeqR+elLlhRz30un6BzwsGZBXnBR+iyng99/7OLglAipwOg1NFwsjrZFkJlmR6nwFsGQ10/GLOzzLUSspc4tZBIZHGNE68VLivH6NXWt/bzLbA1YCrKcE87zP5fkZqQNjyMY9EbVYwiM+klWmn1Ei0BqBCI1SCCYhdxjTH984aIiHDZFltPOzevmJ+LQZo3cDAcefwC310+f2xvVPEOWrHRHWItg0CM1ApEaJDU0Cw16zBrBiBZBltPBbZsqKc3NiDoVkuysUcT9Q76o1yKwGGsSjOg+Ki0CkQIkEMxCwRpBhIvQt9+ydqYPZ1YKnYo6VoHAWJNAppgQqUdSQ7OQrIw1MWuVss4BD4Nef0xaSNnp9uA4An9A4/EHUqonlkhd8imfhWT644lZLYDmHmOK7WiLxWC0CKzWmJwDkUokEMxC1gyYchEaWzAQdLvNn2PUIjD/9oOycL1IIRIIZiFJDU0s10wNNZqrr8WmWOwIdh+VFoFIJRIIZqFBjwSCiVgX/uFAEIsWwXD3UXdwvWI5B2Luk0AwC8nd6MRyRtYIopxiAowZSIdrBEYX3ow4rF0rxGwjn/JZyOWRJRInkma3kZFmo8msEcRiQFl2ugOvX+PxBSQ9J1JKVFcapdR3lVJHlFL7lVJ/UkoVhDz2BaVUnVLqqFLq+pDtW81tdUqpz0fz+nPVoMyDPym5GWl0DnjM72PTIgBjKmq3FItFCon2lvMpYI3Wei1wDPgCgFJqFfBOYDWwFfixUsqulLIDdwM3AKuAd5n7ihAyonVyckPWVojFOgvBNQk8vuE6jQQCkQKiCgRa6ye11taY/FeASvP7W4AHtdZDWutTQB1woflVp7U+qbX2AA+a+4oQgx4JBJNhtQKynfaYrJdsrUng8vhDuo9Kek7MfbH8lH8Q+Kv5/QLgbMhjDea2sbaLEC6PpIYmwyoYx2reJWuZwoEhH0NWsVjOg0gBE7anlVJ/AyoiPPQlrfXD5j5fAnzAA9avRdhfEznw6DFe9w7gDoCFCxdG2mXOkumPJ8caSxCL+gCELE4T0iKQgCxSwYT/g7TWW8Z7XCl1O/BG4BqttXVRbwCqQnarBJrM78faPvJ17wXuBdi0aVPEYDFXyaLpk2MFgLwYrSImxWKRqqLtNbQV+GfgZq21K+ShbcA7lVLpSqlFQC2wE3gVqFVKLVJKOTEKytuiOYa5SHoNTc5waihGLYL00S0CCQQiFUT7P+i/gHTgKaUUwCta649prQ8ppR4CDmOkjO7UWvsBlFKfBJ4A7MB9WutDUR7DnOPy+GVE6yRYtYGY1QisFoHHx6DXj9Nuw26LlOUUYm6JKhBorZeO89i3gG9F2P4Y8Fg0rzvXuT1+suROdEJ5MW4RZI0oFkuPIZEq5JM+C0mxeHKssQOxGFUMw4XhgSG/sUylBGORIiQQzCCvP8DPXjzFNd9/lqMtfWPuJzWCyRlODcWmRWC3KTLTjPmG3D4JxiJ1yFKVM2T7iXa+uu0Qx871A/D4wRaWV+SO2i8Q0LJE4iRZxeJYLEpjsWYgHZSxHCKFSItgBhxp6eUffrIDl8fPve/dyIqKXHad7oy4r9tn9FbJkrvRCeXGeEAZGIvTuIZ8uH0B0iUQiBQhgWAGnGwbAODe927iutUVbKopZO+ZbvyB0cMjZC2CyVtWnstVy0vZWF0Ys+fMchotArfHT6YUi0WKkE/6DOhyGTNkFmU7AdhUXUT/kI8jLb2j9pX+65OXk+7g/g9cSFVRVsyeM9tck2DQK8VikTokEMyAbpcXgIIsI4Wxqca4g91V3zVqX5n1MrGy0h0MDPlldLdIKRIIZkDPoJeMNFvwDnNBQSYVeRnsOh0hEHilRpBI2U47A0M+6bklUor0GpoBXQMeCjKdwZ+VUmysKWRX/eiCsbQIEivL6cDl8TMkxWKRQqRFMAO6B73BtJDlgupCmnvcwcXXLYOyaHpC5aTbGfD4JDUkUooEghnQ7fKMCgSbaooARrUKrBaBpIYSIyvdgcusEcgUEyJVyCd9BnS7vBRmOcO2rajIJdtpH1UwlnnwEyvbacfjD+ALaDkHImVIIJgBkVJDDruNDQsLRxWMJRAkVpZzuGwm3UdFqpBAEGdaa7pdHvIznaMe21hdyJGWXnrd3uA2KzUkNYLEyE4f/rvLORCpQgJBnLk8frx+TWHW6GkQLqgpQmvYe6Y7uE16DSVWaItAzoFIFRII4qx7MHwwWagV84xJ5+rbB4LbBr1+0uyKNLucmkQIaxFIsVikCPmkx1nXgDG9RKTUUEFmGkpBh7kPIFMbJJi0CEQqkkAQZz1miyBSashht5GfmRYMFoBMf5xg1mI3IMVikTokEMTZ8DxDo1sEYExE1zmiRSBjCBIn9G8vgUCkCgkEcWbNPBqpRgBQlDUiEMgSiQmVHdYikP8eIjXIJz3OrNRQfuYYgSBCi0DWIkic0BaBpOhEqpBAEGfdLg+ZafYx7/KLsp10usJbBJIaSpywYrGcB5EiJBDEWZdr9KjiUEXZTroGPGhtrFYm0x8nlt2mgimhDIecB5EaJBDEWbfLO2ahGIxA4Atoegd9gHQfnQ2yzVaBtAhEqpBAEGc9gx4KxqgPwPDylVZ6yC3dRxPOKhinO+S/h0gN8kmPs4lSQ4VWIBgYAsAl3UcTLstpJyPNhlIq0YcixIyQQBBnE6WGioOBwOhdNOjxy2RnCZad7pBWmUgpEgjiSGttpIbGaxFkDbcIAgHNkC8gF6EEM1oEcg5E6pBAEEcD5syj49UIinOGWwRun8w8OhvkZaaFDSwTYq6TT3scdZsF4JGrk4XKcjrISLPROTCES5apnBU+s6U2ODWIEKlAAkEcWReT/HFSQ2BNM+EdXpRGWgQJtbQsN9GHIMSMktRQHFmBYLwWAUBRjpPOgSHc1jKV0iIQQswgCQRx1D04/oRzlsIsJ50ur6xXLIRICAkEcRScgnqcYjEYXUhDawTSIhBCzCQJBHFkFYsnqhEUZjvpGpAWgRAiMSQQxFG3y0uW0076BJOXFWc76R/y0WtOWS0tAiHETJJAEEfdg94J00IwPM1EQ9cgAFlp0plLCDFzogoESqlvKKX2K6VeU0o9qZSab25XSqm7lFJ15uPnh/zO7Uqp4+bX7dG+gdms2+UZd3oJizXNRGO3EQgynBKfhRAzJ9orzne11mu11uuBR4B/NbffANSaX3cA9wAopYqArwAXARcCX1FKFUZ5DLNW9wQTzlms7qWNZotAagRCiJkUVSDQWveG/JgNaPP7W4BfasMrQIFSah5wPfCU1rpTa90FPAVsjeYYZrPuwckFAmuaiaZuCQRCiJkXdTJaKfUt4H1AD3C1uXkBcDZktwZz21jb56TJpoaCLYLuQZx2Gw67pIaEEDNnwiuOUupvSqmDEb5uAdBaf0lrXQU8AHzS+rUIT6XH2R7pde9QSu1SSu1qa2ub3LuZRbTWRmpoEsXigiwnSoHL4w8ukyiEEDNlwhaB1nrLJJ/rN8CjGDWABqAq5LFKoMncftWI7c+O8br3AvcCbNq0KWKwmM0GPH58AT2p1JDdpijITKPL5ZWuo0KIGRdtr6HakB9vBo6Y328D3mf2HtoM9Gitm4EngOuUUoVmkfg6c9uc0zVgTS8xcWoIhpeslPqAEGKmRVsj+Hel1HIgAJwGPmZufwy4EagDXMAHALTWnUqpbwCvmvt9XWvdGeUxzEo9g5ObXsJSlO3kRNsAmU4ZQyCEmFlRXXW01m8dY7sG7hzjsfuA+6J53WQQnGdoyi0CqREIIWaWXHXixGoR5E+6RZAOyPQSQoiZJ4EgTlweHzD51caKso2AkSnTSwghZpgEgjhx+wLA5FcbkxaBECJRJBDEyZDXWnZycn/i4RaBnBIhxMySq06cTHX94WCLQLqPCiFmmASCOHH7/NhtirRJThdRZPYuku6jQoiZJoEgTtzewJTu7otyZECZECIxJBDEids7tXmDirOdFGalUVWUGcejEkKI0SQPESeDXv+ES1SGykiz8/IXrsEpM48KIWaYBII4GfIGpjyT6GQLy0IIEUty+xknbq9fxgQIIZKCBII4cfv8ZEwhNSSEEIkigSBO3N6ApHqEEElBAkGcDMpqY0KIJCFXqjhx+/ykS4tACJEEJBDEydAUB5QJIUSiSCCIk6kOKBNCiESRK1WcDHql15AQIjlIIIgDrbXZIpBAIISY/SQQxIHXrwloWWRGCJEcJBDEgdtnrEWQ7pA/rxBi9pMrVRy4vVNblEYIIRJJAkEcuD1TW69YCCESSQJBHFipIek+KoRIBnKligMrNSQDyoQQyUACQRy4vZIaEkIkDwkEcTDoldSQECJ5yJUqDqzU0FSWqhRCiESRQBAH0n1UCJFMJBDEwZBZI5CRxUKIZCCBIA6C3UdlZLEQIgnIlSoOBj2SGhJCJA8JBHEg3UeFEMlEAkEcuH1+nHYbdptK9KEIIcSEJBDEgdvrJ13GEAghkoRcreLA7Q1IWkgIkTQkEMSBrFcshEgmMblaKaX+j1JKK6VKzJ+VUuoupVSdUmq/Uur8kH1vV0odN79uj8XrzzZuWa9YCJFEHNE+gVKqCrgWOBOy+Qag1vy6CLgHuEgpVQR8BdgEaGC3Umqb1ror2uOYTdxevwwmE0IkjVi0CH4IfA7jwm65BfilNrwCFCil5gHXA09prTvNi/9TwNYYHMOs4vYGpEUghEgaUQUCpdTNQKPWet+IhxYAZ0N+bjC3jbU90nPfoZTapZTa1dbWFs1hzrhB6TUkhEgiE6aGlFJ/AyoiPPQl4IvAdZF+LcI2Pc720Ru1vhe4F2DTpk0R95mt3F4/pbnpiT4MIYSYlAkDgdZ6S6TtSqnzgEXAPqUUQCWwRyl1IcadflXI7pVAk7n9qhHbn53Gcc9qQ76ArE4mhEga085faK0PaK3LtNY1WusajIv8+VrrFmAb8D6z99BmoEdr3Qw8AVynlCpUShVitCaeiP5tzC7SfVQIkUyi7jU0hseAG4E6wAV8AEBr3amU+gbwqrnf17XWnXE6hoQxAoG0CIQQySFmgcBsFVjfa+DOMfa7D7gvVq87Gw1KIBBCJBHJX8SY1trsPip/WiFEcpCrVYwN+cwpqGVAmRAiSUggiDFrmUoZUCaESBYSCGJsUBauF0IkGQkEMeYOBgL50wohkoNcrWLMWrheBpQJIZKFBIIYk/WKhRDJRgJBjFmpIZl0TgiRLORqFWNSLBZCJBsJBDE2ZAUC6T4qhEgSEghizKoRyAplQohkIYEgxqT7qBAi2cjVKsYGJTUkhEgyEghiTLqPCiGSjQSCGAt2H5XZR4UQSUKuVjHm9vlJd9iw2SItzyyEELOPBIIYG/IGJC0khEgqEghibNAj6xULIZKLXLFizO2TZSqFEMlFAkGMub1+mXlUCJFUJBDEmNsbIF0CgRAiiUggiJI/oOlxeYM/u71+WbheCJFU5IoVpfu313PFd/8eHD/g9kqNQAiRXCQQRGn36U56Br2caOsHjNSQ1AiEEMlEAkGUjrT0Gf82G/8avYbkzyqESB5yxYqC2+unvn0AgKPn+oLbJDUkhEgmEgiicPxcPwFtfG+1DIwBZRIIhBDJQwJBFI609AJw/sICjjQb37t9AVmvWAiRVOSKFYUjLX1kpNm4bnUFrX1DdPQP4fFJsVgIkVwkEEThaEsfy8pzWTUvD4D9DT2ArEUghEguEgiicKSll+XluayoyAVg79luABlQJoRIKnLFmqb2/iHa+z0sr8ilNDedwqw0XrMCgbQIhBBJRALBNB01ewmtnJeHUooVFXnsk0AghEhCEgim6XWzl9ByMy20vCKXnkFjziEJBEKIZCKBYJqOtvRRkpNOSU46ACvn5QYfk5HFQohkIlesaTp6ri9YJAZYXpEX/F5aBEKIZBJVIFBKfVUp1aiUes38ujHksS8opeqUUkeVUteHbN9qbqtTSn0+mtdPFH9Ac7SlL5gWAlhWnoMy16uXQCCESCaOGDzHD7XW3wvdoJRaBbwTWA3MB/6mlFpmPnw3cC3QALyqlNqmtT4cg+OYMac7BhjyBcJaBFlOB9VFWdR3uGRAmRAiqcQrNXQL8KDWekhrfQqoAy40v+q01ie11h7gQXPfpGLNK7QiJB0Ew4VjqREIIZJJLFoEn1RKvQ/YBfyT1roLWAC8ErJPg7kN4OyI7RfF4Bgi6nZ5uO2/X47583a5vNgU1JbnhG1fXpHHE4fOSWpICJFUJgwESqm/ARURHvoScA/wDUCb/34f+CCgIuyvidwC0WO87h3AHQALFy6c6DAjstnUqIt1rKyenz/qgn/bxkrQmrLc9Li8phBCxMOEgUBrvWUyT6SU+gnwiPljA1AV8nAl0GR+P9b2ka97L3AvwKZNmyIGi4nkZaTx43dvnM6vTktVURafvW75jL2eEELEQrS9huaF/Phm4KD5/TbgnUqpdKXUIqAW2Am8CtQqpRYppZwYBeVt0RyDEEKI6ERbI/gPpdR6jPROPfBRAK31IaXUQ8BhwAfcqbX2AyilPgk8AdiB+7TWh6I8BiGEEFFQWk8r6zKjNm3apHft2pXowxBCiKSilNqttd400X7Sz1EIIVKcBAIhhEhxEgiEECLFSSAQQogUJ4FACCFSXFL0GlJKtQGno3iKEqA9RoeTLFLxPUNqvu9UfM+Qmu97qu+5WmtdOtFOSREIoqWU2jWZLlRzSSq+Z0jN952K7xlS833H6z1LakgIIVKcBAIhhEhxqRII7k30ASRAKr5nSM33nYrvGVLzfcflPadEjUAIIcTYUqVFIIQQYgxzOhAopbYqpY4qpeqUUp9P9PHEi1KqSin1d6XU60qpQ0qpT5vbi5RSTymljpv/Fib6WGNNKWVXSu1VSj1i/rxIKbXDfM+/M6c7n1OUUgVKqd8rpY6Y5/ziuX6ulVKfMT/bB5VSv1VKZczFc62Uuk8p1aqUOhiyLeK5VYa7zOvbfqXU+dN93TkbCJRSduBu4AZgFfAupdSqxB5V3PgwlgldCWwG7jTf6+eBp7XWtcDT5s9zzaeB10N+/g7wQ/M9dwEfSshRxdd/Ao9rrVcA6zDe/5w910qpBcCngE1a6zUYU9i/k7l5ru8Hto7YNta5vQFjrZdajNUc75nui87ZQABcCNRprU9qrT3Ag8AtCT6muNBaN2ut95jf92FcGBZgvN9fmLv9Arg1MUcYH0qpSuAm4Kfmzwp4A/B7c5e5+J7zgCuAnwForT1a627m+LnGWDslUynlALKAZubgudZaPw90jtg81rm9BfilNrwCFIxYLGzS5nIgWACcDfm5wdw2pymlaoANwA6gXGvdDEawAMoSd2Rx8SPgc0DA/LkY6NZa+8yf5+I5Xwy0AT83U2I/VUplM4fPtda6EfgecAYjAPQAu5n759oy1rmN2TVuLgcCFWHbnO4ipZTKAf4A/KPWujfRxxNPSqk3Aq1a692hmyPsOtfOuQM4H7hHa70BGGAOpYEiMXPitwCLgPlANkZaZKS5dq4nErPP+1wOBA1AVcjPlUBTgo4l7pRSaRhB4AGt9R/NzeespqL5b2uiji8OLgVuVkrVY6T93oDRQigw0wcwN895A9Cgtd5h/vx7jMAwl8/1FuCU1rpNa+0F/ghcwtw/15axzm3MrnFzORC8CtSaPQucGMWlbQk+prgwc+M/A17XWv8g5KFtwO3m97cDD8/0scWL1voLWutKrXUNxrl9Rmv9buDvwNvM3ebUewbQWrcAZ5VSy81N12CsDT5nzzVGSmizUirL/Kxb73lOn+sQY53bbcD7zN5Dm4EeK4U0ZVrrOfsF3AgcA04AX0r08cTxfV6G0STcD7xmft2IkTN/Gjhu/luU6GON0/u/CnjE/H4xsBOoA/4XSE/08cXh/a4Hdpnn+89A4Vw/18DXgCPAQeBXQPpcPNfAbzHqIF6MO/4PjXVuMVJDd5vXtwMYvaqm9boyslgIIVLcXE4NCSGEmAQJBEIIkeIkEAghRIqTQCCEEClOAoEQQqQ4CQRCCJHiJBAIIUSKk0AghBAp7v8B6Fk0Yv6HSzUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "actor_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.0001}\n",
    "critic_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.0002}\n",
    "agent = AdvantageActorCriticAgent(env, actor_kwargs=actor_kwargs,\n",
    "        critic_kwargs=critic_kwargs)\n",
    "\n",
    "def play_qlearning(env, agent, train=False, render=False):\n",
    "    episode_reward = 0\n",
    "    observation = env.reset()\n",
    "    step = 0\n",
    "    while True:\n",
    "        if render:\n",
    "            env.render()\n",
    "        action = agent.decide(observation)\n",
    "        next_observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        if train:\n",
    "            agent.learn(observation, action, reward, next_observation,\n",
    "                    done)\n",
    "        if done:\n",
    "            break\n",
    "        step += 1\n",
    "        observation = next_observation\n",
    "    return episode_reward\n",
    "\n",
    "# 训练\n",
    "episodes = 100\n",
    "episode_rewards = []\n",
    "for episode in range(episodes):\n",
    "    episode_reward = play_qlearning(env, agent, train=True)\n",
    "    episode_rewards.append(episode_reward)\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "# 测试\n",
    "episode_rewards = [play_qlearning(env, agent) for _ in range(100)]\n",
    "print('平均回合奖励 = {} / {} = {}'.format(sum(episode_rewards),\n",
    "        len(episode_rewards), np.mean(episode_rewards)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "带资格迹的执行者/评论者算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ElibilityTraceActorCriticAgent(QActorCriticAgent):\n",
    "    def __init__(self, env, actor_kwargs, critic_kwargs, gamma=0.99,\n",
    "            actor_lambda=0.9, critic_lambda=0.9):\n",
    "        observation_dim = env.observation_space.shape[0]\n",
    "        self.action_n = env.action_space.n\n",
    "        self.actor_lambda = actor_lambda\n",
    "        self.critic_lambda = critic_lambda\n",
    "        self.gamma = gamma\n",
    "        self.discount = 1.\n",
    "\n",
    "        self.actor_net = self.build_network(input_size=observation_dim,\n",
    "                output_size=self.action_n, output_activation=tf.nn.softmax,\n",
    "                **actor_kwargs)\n",
    "        self.critic_net = self.build_network(input_size=observation_dim,\n",
    "                output_size=1, **critic_kwargs)\n",
    "        self.actor_traces = [np.zeros_like(weight) for weight in\n",
    "                self.actor_net.get_weights()]\n",
    "        self.critic_traces = [np.zeros_like(weight) for weight in\n",
    "                self.critic_net.get_weights()]\n",
    "    \n",
    "    def learn(self, observation, action, reward, next_observation, done):\n",
    "        q =  self.critic_net.predict(observation[np.newaxis])[0, 0]\n",
    "        u = reward + (1. - done) * self.gamma * \\\n",
    "                self.critic_net.predict(next_observation[np.newaxis])[0, 0]\n",
    "        td_error = u - q\n",
    "        \n",
    "        # 训练执行者网络\n",
    "        x_tensor = tf.convert_to_tensor(observation[np.newaxis],\n",
    "                dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            pi_tensor = self.actor_net(x_tensor)\n",
    "            logpi_tensor = tf.math.log(tf.clip_by_value(pi_tensor, 1e-6, 1.))\n",
    "            logpi_pick_tensor = logpi_tensor[0, action]\n",
    "        grad_tensors = tape.gradient(logpi_pick_tensor,\n",
    "                self.actor_net.variables)\n",
    "        self.actor_traces = [self.gamma * self.actor_lambda * trace +\n",
    "                self.discount * grad.numpy() for trace, grad in\n",
    "                zip(self.actor_traces, grad_tensors)]\n",
    "        actor_grads = [tf.convert_to_tensor(-td_error * trace,\n",
    "                dtype=tf.float32) for trace in self.actor_traces]\n",
    "        actor_grads_and_vars = tuple(zip(actor_grads,\n",
    "                self.actor_net.variables))\n",
    "        self.actor_net.optimizer.apply_gradients(actor_grads_and_vars)\n",
    "        \n",
    "        # 训练评论者网络\n",
    "        with tf.GradientTape() as tape:\n",
    "            v_tensor = self.critic_net(x_tensor)\n",
    "        grad_tensors = tape.gradient(v_tensor, self.critic_net.variables)\n",
    "        self.critic_traces = [self.gamma * self.critic_lambda * trace +\n",
    "                self.discount* grad.numpy() for trace, grad in\n",
    "                zip(self.critic_traces, grad_tensors)]\n",
    "        critic_grads = [tf.convert_to_tensor(-td_error * trace,\n",
    "                dtype=tf.float32) for trace in self.critic_traces]\n",
    "        critic_grads_and_vars = tuple(zip(critic_grads,\n",
    "                self.critic_net.variables))\n",
    "        self.critic_net.optimizer.apply_gradients(critic_grads_and_vars)\n",
    "        \n",
    "        if done:\n",
    "            # 下一回合重置资格迹\n",
    "            self.actor_traces = [np.zeros_like(weight) for weight\n",
    "                    in self.actor_net.get_weights()]\n",
    "            self.critic_traces = [np.zeros_like(weight) for weight\n",
    "                    in self.critic_net.get_weights()]\n",
    "            # 为下一回合重置累积折扣\n",
    "            self.discount = 1.\n",
    "        else:\n",
    "            self.discount *= self.gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均回合奖励 = -9103.0 / 100 = -91.03\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8W9X9//HXkWXJS94jTmzHgTib7AFhhwABWiiFtkAp0PGj/Ra6F3xp+XbxLd27tBQopV8KBcpI2YGwA9k7cRJnOR7x3kPz/P7QlSzZ8pQdj/t5Ph55xLqSpSvLvu97zuecc5XWGiGEEOZlGe0dEEIIMbokCIQQwuQkCIQQwuQkCIQQwuQkCIQQwuQkCIQQwuQkCIQQwuQkCIQQwuQkCIQQwuSso70DA5GZmakLCwtHezeEEGJc2bp1a63WOqu/x42LICgsLGTLli2jvRtCCDGuKKWOD+Rx0jUkhBAmJ0EghBAmJ0EghBAmJ0EghBAmJ0EghBAmJ0EghBAmJ0EghBAmNy7mEQjRH601a3dWsGRqGnlpCaO2H+0uD89sL+cTS/OxxpjzPGvr8QbeOlAdvL2wIJVVs3IG9L2N7S7WF1dz1cIpxFjUSO2i6EaCQIyq/ZXN7C5r4uPL8qN6nsM1rXzl8R0kx1n56TXzueyM3GHaw8H59bqD/PWdo0xOjefCmdmjsg+j7Scv7mfL8QaUAq0hwRbD5rtWk2jv+3Dj9Wlu/+d23i2ppdPt44YVBadoj4U5T1nEmHHfm4e54+lddLi8UT3PvsoWANITbfzXo9v47rO76XRH95yDday2jYc3HANge2njKX3tseRYXRvXLcvn6E+u4MkvnEW7y8tLe072+31/WF/CuyW1ZDns/GrdAVo63adgbwVIEIgh8Po033pyJxsO10b9XNtPNODTcKCqJarnKa5sxmpRvPDlc7n1vNP4vw9KefDdo1Hv32D85KX9xMZYyEuLZ3tpwyl97bGi1emhttXF1IxEAJZOTaMwI4Ent5wIe1xLp5sNh2uDYb2hpJbfvH6Qjy6awoM3L6W21cUf3zh8yvc/oN3l4WCUv5N90Vpz1zO7eWFX5Yi9xmBIEJhIc6ebdw9Ff/B+80A1T24t45ENA1rGpFc1LU5O1HcAsLeiKarnOnCyhenZSSTarfz35bOZOzmZtw7WRPWcfdlwuJZr7tvAun1VaK15/3Adr+yt4osXnM55M7LYcaIRn0+P2OuPVcfr2gCYmuGv0yiluHZJHhuP1lNa1x583Nef2MkNf93Iwh++ymce3syXH9/B6VlJ/PjqeczPS+Wji6fw0LtHOVHfHvF1Rtq9LxVz6W/e5rkd5SPy/PsrW3h0YynfemrnqL3HUBIEJvLE5hPc+OBGqpo7o3qev7/vD4ANh2vxeH1Dfp4dJ7q6T/ZVNEe1T8UnW5g5yRG8ffb0TLaXNkTd5QREPKA//N4xth5v4P89soVb/raZH/xnL5NT4vjcuaexKD+Vlk4PR2pbo37tscbbT7gFDvYF6V0F+48uzkMpeGpbGQDr9lWxbl8VnzpzKtctK6CkupVOt5c/3rCYBJu/jvCtS2discC9LxcPeN8a2lw8uvF42O9Vf7TWaK17bFu3rwqFP7Be2j38Z+3P7SjHalFYlOKbT+4c9ZMGCQITqWl1ArCrbOBn361OT9jtIzWtvH2whjm5yTR3ethV3vtzdbq9OD29H4i3lzZgtSgW5Keyr3LoQdDU4aa8sYNZk5KD21aenoHbq9l8rH7Iz+v2+rjtn9u4/HfvhB0A210e3jpYww0rCvjuFbPZdryB4pMtfOeyWcTFxrCoIA2AbSF1ArfXxx/WH6LW+AxGS1OHm1+9eiB45j4Y1S2dnPezN7jz6V09Dp4Bx4wgCLQIACanxnPO9Ez+vbWMVqeH76/dy4ycJO7+8By+f+Vc3vrWBWz73sVhQZ6bEs/nzzudF3ZVcusjW9h0tL7X1zxc08pdz+zmrHtf565n9vCRP77Ht57cSU1L7z/r5k43D7xzhHN++gbX//WDsPv2V7ZQ2dTJ9z40h4X5qXzpse28tq9qwD+n/nh9mud2VHD+jCzu/vAcNh6tD9aWRosEgYk0tfuLb7v7OHiH2l7awOIfruNX6w4Gtz3y/nFiYxS/uW4hStFnV9Nn/76Zrz+xs4/nb2TO5GQWF6RSXNnS79lmbwJ9ubNyuw4ky6elY7Uo3uujjlHd0hn8mXTn8fr46uM7eGFXJcUnW3jrYNdwyDcP1OD0+Pjw/Ml87tzTeP2b5/PnGxdz5YLJAJyWmUhynDWsYLxuXxW/ePUg6/dX93itE/XtuDw9W1Z/eeswf37r8LAWvdfurOB360u4+Ndv88tXDwy4xeT1ab7y2A4qmjp4bNMJ/vRm5P770vo2MhJtOOJiw7ZfuySP8sYOPvvwZsobO7jn6jOINYbXKqWwWXseir544el8adV0Nh2r5+N/eZ8P/+Fd/vLWYQ5VtaC15t1DtXz6b5u46Jdv8eTWMq5cMJlnvriSz59/Gs/uKGfVL97k9f09D+D/+OA4K3+ynh+/sB+AD47UU3yy60RkfbH/e66Yn8vfPr2MOZOT+eI/t3GoW81Aa82RmtZeA6o3G4/WcbK5k6sWTeFjS/K4aFY2P325mMM1PVuQv3r1AD96ft+gnn8oJAhMpKHdBcCeAQSB1pofPb8Pl9fH714/xDPb/Wdz/95axhVn5DIjx8Hcycm9BkFDm4sNh+vY0ssZuden2VnWyKL8VObkJtPh9nK0dvBnqeAvFAPMCjmjTLBZWVSQyoaSul6/78YHNnLNnzfQ1q3V4/VpvvXULl7YXcm318wky2HnnxtLg/e/tOck6Yk2lhX6z/yzHXGsmZeLUv5x7xaLYmFBWljBOFAsben2Wj6f5rLfvsPX/rUj7ICyoaSWn7xUzL0vFXPJr9/mNaMW0V1dqzPiAaQ3+yqaSImP5fJ5k/j9+hJW/+otqlt6dhVuK22gLqT18tvXDvL+kTp+ds18PrJwMj9/5QAvRugyOV7XHtYaCLh07iQcdisbj9bziaX5LCtM73df7dYYvnHJTN6/4yLuuXoePh/85KViLv7128z/wavc+OBGdpc38dXVRWy4YxU/u3YBiwrSuPOy2bz81fPISYnjnhf2h3W7NLW7+d8X9jMnN5n/3H4O//nSOcTGKP69tSz4mPXF1czPSyHbEUdyXCwP3ryMJLuVbzy5E3dIV+iPX9jPql++xQ+f3zeok5jntleQaIvh4tk5KKX4yUfPwG618OuQEy7w/w3+e1v5KakhSBCYSOMgWgTP76pkW2kjP/7IPM46LYPvPLWb76/dS4vTw00rCwE4tyiLbaUNPbqPAN4+VIPWUNXspKHN1eP+g1UttLu8LCpIY+7kFIAhdw/tP9lCSnwsk5LjwravPD2TPRVNEc/6S6pbOVjVSkl1K997dk/wIOv2+vjWUzt5Zns537xkBl+8YDofX5rH+uJqKps66HR7Wb+/ikvm5PQ5YWxRfioHq1poc3qobu4MFq5bO8N/Vu1uL61ODy/sruQ/xggSp8fLd5/dw9SMBB66ZSk2q4XPPbKFr/1rR1hNpr7NxdV/2sCHf/8u1QOs++wpb+aMKSn85rpF/O2WZZQ3dvBGcXgrpbSunY/+aQMr713PnU/v5vFNpfz+jRI+tiSPjy3N595r5rNkahpf+9cOdpWF98f7gyCxx+vGxcZwzZI8MpPs3HHZrAHta0C8LYZPrpjKi185lw13rOKeq+exenYOP7t2Pu9+ZxVfXT2DzCR72PecnpXEbReezpHaNt4t6TpZeXLrCTrcXv7nyjmckZdCeqKNi2bl8Mz2ctxeH3WtTrafaGTVrK45IFkOOz/+yDx2lTVxn9ESenTjcR589yizJjn423vHuP2f2wbUcut0e3lxdyWXzptEvC0GgOzkONbMm8TbB2vCPt9jde2UN3ZwblHmoH5eQyFBMEJ2nGjk3J+tj7owO1DbSxt4eltZn2cmTR3+A2JNi7PP/ep0e7n3pWLm5CZz/fIC7rtxMXlp8Ty1tYz5eSksyk8F4NzpmXh8mg8O9zzrDh2xU3yy5zC8QLfJooJUpmcnERujwgrG64urWPWLN2l3hR84j9e1cfa968OG9hVXNjNzkiN4Rh5w9vRMtIb3j/Tcv3VGn+8NKwp4ens5T2w5QavTw2ce3szT28r5xsUzuH1VEQDXLSvAp+GJzWW8V1JLm8vLmnmTev35Bd6XT/vrMU9vL8enwaLoMTY+cNtqUdz93B6qmzv5y1tHOFLbxg+vmseqWTm89JVz+erqIp7dUcE3n9yJ16dxerx8/h9bONncidvr4xevHuhzf8AfcgdOtjB3sr+Wcv6MLBxxVnacCD8x2Frqb8VdODObp7eVccfTu5mR7eCHV80D/Af1+z+1hOT4WH6/viT4fU6Pl4qmjogtAoC7rpjNm9+6gLREW7/72pvJqfF8csVUfv2JhXx8aT5xsTG9PvbyM3LJTLLxd6P/3efT/OOD4ywr7Dr5AH+3VW2ri7cO1PDWQf8JTGgQBJ7rygWT+d3rh3jgnSPc/dxeLpyZxQtfPpfvXjGbl/ee5MYHNkYMA4/XFzzReKO4mhanh48snBL2mPNnZNPc6WFnSLC+e8j/N3RuUb9XmoyaBMEIeedgDSfqO3h1GItMvWlzevj8P7by9Sd28uHfv9trd0xju5vTMv1na7tDCsYNbS5+/Pw+nttRTmO7iwffPUp5YwffvWI2MRZFaoKNB29ZxswcB1+5qCh4wF1SmEZcrIV3DoUP0/T5NG8frGXl6RkAYf2vAdtLG0hPtFGQnoDNaqEo2xHWIvjzm/6D4fG68GbxzrImyhs7gsU1n09zsKqV2SHdQgEL81OJj42JON9h3b6TzJuSzI+umsc50zO5+7m9XPOnDWw47O/++NJFRcHH5qcncG5RJv/aXMoLuypxxFlZeXrfZ2kLjbDcVtrAU1vLWDo1jYwke4/WU4vRQvjyRUV0uLzc9s9t/OGNEj40P5fzZ/gPALExFr66egbfunQmz+6o4I5/7+LOp3ez+VgDv/jYAm4+q5Ant5b12+V3qKoVl9fH3Cn+g6DFoliYn8rObqNstpc2kmiL4Y+fXMz7d17ED66cywM3Lw2ewQJkJNm5YEYWW451FXFP1HegNb0GQWyMhaR+ZhcPJ7s1huuXF7D+QDWlde28dbCG43Xt3HRWYdjjzp+ZRWaSjae2lvF6cTVZDjvzQoIi4IdXzSUt0caPX9jP9Kwkfnf9ImIsis+dexo/vWY+W4438OaB8L+F5k43i360jvN//ibfX7uXh947SmaSPfi3EXDO9EwsCt4K+f63D9WSlxbf689zOEkQjJDAWfD6CMWq4faXtw5T3eLk6xfPoKHdxbV/fp+fvLS/x+Ma2l2cdXoGSoV3Dz268TgPvHuUrzy+g8U/WsdvXjvI6tk5rJzedbCblpnIK187j4tmd60ZY7fGsGJaBu+UhB9o91U2U9vq5JrFeWQk2iiujNAiOOGvDwRCZc7kZPZVNKG1pqS6hU1GmFU2dYR930nj9nPby2lzeihv7KDV6WFWbjLd2awWlk9LZ0O3FktNi7/5f/HsScRY/IXvlPhYTjS08+DNSyMud3HD8gIqmjp5Zkc5q2fnRCxuhkpNsHFaViKPbSqlpLqVa5fk4Yiz9qgRBIJgQX4q314zi83HGrDFWPjeh+b0eM7bLpzOly8q4smtZTy9rZyvXzyDKxdM5ksXFZEaH8s9L+zvs3AZmKsRaBEALMhL5UBVS1jReHtpIwvyU4mxKNITbdy8spD89J4Ho2WF6TS0uzlc46/tlNb7/y9I79k1NFo+uWIqFqX4xwfHeHjDMbId9h6tudgYC1cvmsLrxVW8daCGC2dmYYmwzlFqgo3ffGIhZ56WzoO3LA0riF9hLGlSUh3+u76/opmWTg8p8bE8vrmUzccauGrh5B7diikJsSwuSONNoyXt9vr44HAd5xZl9WjpjgQJghESOAvecLiuz5EZW47V9znEEmBXWSP3vXmYbz+1k4//5X1++eqBYBdQRWMH979zhA8vmMyXLyri9W+cz6pZ2fzf++GTvfxDOX1MTo1nelZS8OxRa80z28tZVpjGs7edze0XTufs6ZncHeFAFMm5RZkcqWmjvLHrgB3oFjpvRhazch09WgRNHW5KqltZVJAa3DZ3cjK1rS5qWpw8tqlrFmplU3gXVkWj/3aby8t/dlaw32hFzIzQIgD/MNKS6tawrrDX91ehNVw8xx9qmUl2nr3tbF756nlc0Mv6QKvn5JCZZEdr+u0WCliUn0ZZQwdxsRaumJ+Lw24NHvgDAl1Djjgrn15ZyC0rC/n5tfPJ6VbvCPja6iK+s2YW/3WBf0QNQEp8LF9dPYP3j9QFu7wi2VvRTKIthmkhffgL8lPx+jR7jJDocHnZX9kc9tn0ZqlRLA+0QAOtt8JTcAY7UJNS4lgzdxL/3FjKWwdr+OSKqcHRSqGuXZKP26tpdXr6XCDv7OmZPH7rWT0WNky0W8lLi+dgVXjhPtCFef9NS9hx9yU8fuuZfP3iGRGf+/wZWewqa6K21cnOE420OD2npD4AUQaBUupjSqm9SimfUmppt/vuVEqVKKUOKKUuDdm+xthWopS6I5rXH6s6jREwiwpScXp8vS7FsLeiiWv//D6PbzoR8X7wj2D51IOb+OnLxawvrqG108Pv15cEi1M/f+UAPg3fWTMT8I+WWTI1jTaXNyyAAoXitAQbZ0xJCbYI9lY0c7imjY8smsLC/FS+fslMHv70cgoG+Mcc6L98J6Qm8OaBauZNSSbLYWdmTjIHq1rDaheBrojAeHuAOcYZ/bbSRv69rYw1c/1n65WN4UFwsqmT6dlJzMxx8NimUg4YLa+ZOZGD4GyjVfNeSKtl3b4qpqTGMztkuOnk1PiIZ70BsTEWPnXmVNITbZw3wD7bwMH0snm5OOJiccTF0tqjRuAPBofdisWi+P6Vc/tcME8pxX9dcDrfWTMr7EzxhhUFnJ6VyDee3MmnHtzI3c/t4ZW94ev77ClvYnZuctjZ7oJ8fxdI4DPZU9GEx6dZlJ9Gf6ZlJpKRaGPzMf/oqON17STZraRHUQMYCTevLKTN5SU2RnH9isiLG86c5GB+XgqxMYpzhnjwLcpO4lB19yBoxWG3Mik5jrjYGM48LaPXxffOn+n/vXr3UC3vHKrFoujRhTRSom0R7AE+CrwdulEpNQe4DpgLrAH+pJSKUUrFAH8ELgPmANcbj51QDlW14tNwy8pCEmwxrC/uOXYc4KXd/j/UDyIUMwN2lzfR1OHmN59YyJbvrubFr/iLUy/tOclH/7SBZ7aX87lzpoWdoWQm+f8Q69q6hv81dvhH7qQmxDJvSgrVRsH42e3lxMaoYNN2sGbkJFGYkcDPXzlASXUrTR1utpU2csEM/5n1rFwHHW4vpSFD4LaXNqIUzM/r6oedbXRX/Oa1gzS2u7nxzKnkOOw9WgSVTR3kpsRx/fJ8dpY18dzOCqZmJPT6xzUnN5kpqfHc+1Ixx2rbaHd5eLeklovn5Ay6yf2lVdN559sXhvWV9+Wc6Zk47FZuPHMqAEl2a681gu7j7gcrNsbCH25YzKpZ2TR1uHlmWzmf/8fWYFD6fJp9lc3MmxLe953tiGNKanxwNm5gyOvCAbQIlFIsmZrGluOBFkEbUzMSTklXxmAsK0xjWWEaH1uaT7YjcksL4PtXzuWn18wfch2jKMfB4Zrwk54DVS3MiDCQIZJ5k1PISLTx5oFq3jlUwxl5qaQmnJpQjSoItNb7tdaRhitcBTyutXZqrY8CJcBy41+J1vqI1toFPG48dkIJdIWcMSWFc6Zn8kZxdcS+25eNM7a+Zk0GWhOhZymfO/c0/nDDIkqqW8lMsvPFC6eHfU9gKF1da9ewzYY2/5loanwsZ+R1nQWu3VnBBTOzh/wLp5TiwVuWoZTihr9+wD83luL16eDZzWxjtm9xSCH4jQPVzJ2cHHbwS46LpSA9geKTLRSkJ7Dy9AwmpcT1qBFUNnWSmxLH1YvysFstlFS39toaAH9B9KFbluH2+vjkAxt5fNMJnB4fl8wZ2Pr43Z+rv6WUQxVmJrL7B5eyZKr/7DoprmfXUKuzq2soWrNzk/ntdYtYe/s5vPXtC7HFWHhsk3/+w9G6Ntpd3rD6QMCC/JTgaJXtpY0UpCf0GI7Zm2WF6Ryva6e6pbPXOQSjTSnFk19Yyf9efUafj1tckMZHF+cN+XWmZyfh8viCJz1aaw5VtTAjJ2lA32+xKM6bkcUbB2rYWdbEeaeoWwhGrkYwBQjt7ygztvW2fUIpPtlCXKyFqRmJrJqVTUVTZ48hlCXVLZRUt3LGlBTq2lzBglt3G0rqmDXJ0eMP80PzJ/OfL53DY/9vRY8zmAzjsaHLGTQFWwQ25uQmoxQ88M5RqlucPYayDdbpWUk8+rkVuL0+fvpyMY44a3CIaVFOEhbVVTw/WtvGjhONfHj+5B7PE+geun55ARaLIjc1npMhLQKXx0dNq5PclHhSEmK5Yr6/FROpUBxq5iQH//jsClo63fzw+X0kx1lZNq3/CU3DzRFn7TGPoKXTg0X51+wfTumJNi6dN4mnt5XR6fay1xiaOzfCaJgFeamcqO+grtXJjhONA6oPBATqBJuO1nOioX1MFYpPtRnGCUlgBnJtq4uGdndw+0CcPyOLpg43Xp/mnOljKAiUUq8ppfZE+NfXmXykdpDuY3uk171VKbVFKbWlpmbkVpEcCcUnm5mZ4yDGorjQGI/cvXso0C101xWzAf8fUnedbi+bj9X3OlRx5iQHRRF+yTKMPtrQFkGgRpCaEEui3crpWUlsOlaPw27lotnRX0AlcLBNjrNy8eyuyVZxsTEUZiYGW0nP7ShHKbhyYc8gWDLVPxz12iX+s7Lc5DgqmzqDraXqlk60htwUf/P+xjOnohTB0OnLvCkpPPJZf2hefkZuxILhSHPYrbS6PGEzXVs6PSTZrSPSnXL98nyaOz28uLuSveVN2GIsFEU4Ow0MdX1570kqmzoH9PMMmDs5hbhYC2t3VOD26jFVKD7Vpmf7f7aBOkGgUDyYIDi3KBOlINEWE1ZDG2n9tke11quH8LxlQGhVJg+oML7ubXv3170fuB9g6dKl42o93wMnW4ITUnKS45g3JZn1xdXcFtKF8/LekywuSGXFtHSyHHY2Ha3rcUWmbaUNOD2+QReMAq2H2pAaQUNIEIC/26qkupU18yb1OSlnMOZNSeG9O1ZhtYQfZGdPSmaPMTT02e3lnDktg9yU+B7ff8vZhVy1cDJZDv/+56bG0+H20tThJjXBFqwX5Kb6v3dxQRrvfmcVk1N67/cNtTA/lffuWIW9n6GfIyUpzorW0ObyBLvFWjo9UdcHenPWaRkUZiTw2KZS7NYYZk5yRAzAeVNSsCiCE68GcwCyWS0syEvlDePSlAMdZDARJdmtTE6JC7YIAkEQKXx7k5Fk58xpGWQ57P0OUR5OI/VKa4HrlFJ2pdQ0oAjYBGwGipRS05RSNvwF5bUjtA+joqbFSW2rK2wlzFWzcthe2hBcBqC0rp29Fc1cZqxPs3xaOhsj1Ak2lNQRY1GsOG1w3RjxthgSbTHhLYIOF7YYC/HGQf8Mo2j4kUXD2zPniIvtUUydOclBaX077x+p41hdO1f38pqxMRayQ4ZNBs78AwEQDIKQA/+U1PhBnU2nxMcOW/ANVuCAH1owbul0D0t9IBKlFNcvL2DzsQY2H6tn3pTIXWiJdiszchwcrGrFZrUwu5+utu6WFabj9vp/dwsjLC9hJkU5jrAWQVpCLFkDrLcE/P0zy/nlxxeMxO71Ktrho1crpcqAs4AXlFKvAGit9wJPAPuAl4HbtNZerbUHuB14BdgPPGE8dsIIjNIIXQnzw/NzscZYuO6vH3Civp2X9/rXlAmMR18xLZ3Kpk7KGsILoxsO1zI/L2VIZ4wZSfbwGkG7m9SE2OBB8+PL8vnVxxeckuFpsyY50Bp+9vIBbFYLa84Y2Dj8riDw/1wCk8lyB9gCGGsCtZzQOoG/RTBys22vWZJHbIzC6fFFrA8EBLqHzpiSMugz0UCdwGa19FjvyWyKspMoqfaPHDpY1UpRzsBGDIWyWS2nvOsy2lFDz2it87TWdq11jtb60pD77tFan661nqm1filk+4ta6xnGffdE8/pjUaAvPLRFUJTj4P8+u4LaFidX/2kDj286wdzJycFx68uNwmVonaCl083OsibO7mcpg95kJNl61AgC3ULgPyj5Lxgy8kP9Aj+LHScauWhWNskDDLZA91GgJVDR2EmS3TpiXSkjLck44DeHBoHTPaLvJzPJziVz/cEbacRQwAIjCAZTHwhYPDUNpfwXo4k0I9dMinKScHp8lDW0c/BkS58j2sYSmVk8zPZXtpDtsPeYVLN8Wjr//q+V2K0WjtS2cVnI7NQZ2Q5S4mPDgmDT0Xq8Ps3K6UM7Y89IDG8RNLS7SI0fnYk+eWnxJBrdRYPpispy2MMmlZ00ho6OV8lGEIR2DbWOcIsA4MurirhmcV6fLYJlhen+CUxD+H1LjosNLidudtOz/Qf+tw/V0uL0DHjo6Gg7dStAmcSBquZehzMW5Th4+osrefDdo9ywYmpwu8WiWFaYzsajXRPL3iupw261sHiIIweyHLawlQybOtxhlw88lSwWxcxJDkqqW7lg5sBXUoyxqLBJZZVNHUwax0GQZDdqBN26hkZ6IbaZkxz99jlPz07i/TsvItsxuP7sgIc/sxyryVsD0FUYfmFXhXF7fLQIJAiGkcfr42BVK7es7L07Jyc5jv++fHaP7SumpfPa/ioqmzqoaOzg9eIqlhamDbmwmZFop77Nhc+nsVgUje1u5ueNXpfKnZfPpqXTjd06uPcTOqmssqkzrMttvAmc+YcuRT2So4YGq7f1jQZioN19E11ynP+6GIHW/WCGjo4mCYJhdKyuDZfHF3alrIEK1Aku+PmbOD0+YiyKb14yc8j7kpFkw+vTNHa4SU+00djhOmXT1SMZyBWpIslNjWd/RXNwMtm4bhF06xpyery4vL4R7xoSp1ZRThInmzvJTOrZRTxWyW/gMAp2eCx1AAAXVklEQVRcbGUoZ61zJydz8ZwcHHYrq2Znc25RFinxQz/L6lpmwkmCLYZOty+q5xstuclxvL6/iqpm/2SyyanjNwgSbeHF4q51huTPcCIpynbwzqFaZk4aH/UBkCAYNkdqWvnxC/uZkZM0qAkkAdYYC3+9aWn/DxygDGPhudpWV7DrIW0UWwRDlZsaT6fbF1yiYlKEiWjjRYxF+ReekyCY0AJ//0XZ46NbCGTU0LBoaHPxmYc3Y7UoHrx52agsX9BdZsh6Q6Erj443gVFC24xVMQc6i3is8q9A6q8RtAaXoB5/n4voXZGx1MR4qQ+ABEHUXB4fX/i/rVQ0dXL/TUv6XNP+VOpab8jZtc7QeOwaMg78geWRx3ONAMJXIA0UjZOkRTChLC5I4+4PzYm4ntZYJb+BUXp043E2Hq3nt9ctZMnUU7+iZW/SEmxYFNS1uWhs97cIUsZli8DfFbTzRBOOcTyZLMAR13VNgmbpGpqQLBbFZ86ZNtq7MSjSIohSSXUraQmxXBXlUs7DzWJRpCfaqW11hV2dbLwJTCrrcHvHfWsA/F1D3VsEMvRSjDYJgihVNXdGNf56JGUm2YwaQfjKo+NJYFIZdK06Op454qzBAAi0DKRFIEabBEGUTjZ3jtkzVf96Q/4aQejKo+NN4OebO0YDdzAc9thgAARaBoO56pkQI0GCIEonm5xjdsXFzCR7sEYQuvLoeBNoCeSO4zkEAUlxocNH3cTHxoyJUWbC3OQ3MApur4+6NueY7RrKSLRTZ9QIxmO3UECgJTCeF5wLcMRZaXN58fo0rc6RX3BOiIGQIIhCdYsTrcfukMaMJButTg8nmztHbeXR4RBsEYzjyWQBwWsSOD00d3pk6KgYEyQIohC4UMpY7RoKXBnpcE3ruBw6GjB7kgOrRXF69viZst8bR8h6Q2NpwTlhbnI6EoWTTf71/sds15CxzERLp4e0cRwEK6dnsvV7F4/LtZK667pWsZuWTnfwGgVCjCZpEUThZHPPa+iOJRkh10odzZVHh8NECAEIv1zlqbgojRADIUEQharmTmxWy5gtxGaELIE7UQ6k412gJtBidA2N9EVphBgICYIonGzqZFJy3JgdlpkZ0iIYj7OKJ6Lk4MVpPLR0juz1ioUYKAmCKJxs7hyzhWKAeFtM8FrBY7XVYjaBy1U2dbhpc3mla0iMCRIEUahq7iRnjNYHAgJ1gvG48uhEFOgaqjKuwyxdQ2IskCAYIq210TU0tIt9nyqBkUPjefjoRJJoi0EpqGj0Dz2WBefEWCBBMERNHW6cHt+YHToaEKgTSI1gbFDKf5WyCmMOinQNibFAgmCIuoaOju3ZrplGi0BqBGOHw26lMtA1JEEgxgAJgiE6afwhT0oZ211DRdkOclPixu3KoxORIy42GAQyakiMBXI6MkRVRotgrHcN3byykBtWFIzZIa5mlBRnxeXxAdI1JMYG+S0cosAZXbZjbAdBjEURY5HWwFgSevCXIBBjgXQNDVFVcyeZSTZsVvkRisEJHTLqsEvXkBh9chQboOrmTg5WtQRvn2wau5eoFGNboBVgtSjiYuVPUIw++S0coHtfLuba+zbQ7vJfXepk89i9MpkY2wIFYkecVWo3YkyQIBig8oYOmjs9PLO9HBgfs4rF2BToGpKho2KskCAYoJpW/7UH/r7hGJ1uL/VtrglxMXVx6gWCQOoDYqyQIBigmmYnmUk2Dla1snZnBYC0CMSQBGoEMmJIjBUSBAPQ4fLS4vRww/ICUhNi+e1rh4Cxe4lKMbZJEIixRoJgAGpa/N1C+ekJfGJZPuXGgmFj9aL1YmwLLEUts4rFWBFVECilfq6UKlZK7VJKPaOUSg25706lVIlS6oBS6tKQ7WuMbSVKqTuief1TpabVP3ksy2HnxhVTsRgDPWT4qBgKaRGIsSbaFsE6YJ7Wej5wELgTQCk1B7gOmAusAf6klIpRSsUAfwQuA+YA1xuPHdMCLYIsh5389ARWz87BEWeVC4+LIUmSIBBjTFS/iVrrV0NufgBca3x9FfC41toJHFVKlQDLjftKtNZHAJRSjxuP3RfNfoy0aiMIAstJ/OSjZ1DW0CFjwMWQBAIgSUYNiTFiOE9JPgP8y/h6Cv5gCCgztgGc6LZ9xTDuw4ioaXFiUZBuXAw+I8kevPKXEIOVmWjnlpWFXDQ7e7R3RQhgAEGglHoNmBThrru01s8Zj7kL8ACPBr4twuM1kbuidC+veytwK0BBQUF/uzmiqpudZCbZibFIC0BEz2JRfP/KuaO9G0IE9RsEWuvVfd2vlLoZ+BBwkdY6cFAvA/JDHpYHVBhf97a9++veD9wPsHTp0ohhcarUtDrJckgLQAgxMUU7amgN8B3gSq11e8hda4HrlFJ2pdQ0oAjYBGwGipRS05RSNvwF5bXR7MOpUN3SSbYEgRBigoq2RvAHwA6sMwqnH2itv6C13quUegJ/EdgD3Ka19gIopW4HXgFigIe01nuj3IcRV9PiZE5u8mjvhhBCjIhoRw1N7+O+e4B7Imx/EXgxmtc9lXw+TW2rS7qGhBATlsws7kd9uwuvT4/5K5EJIcRQSRD0I3QymRBCTEQSBP3omkwmQSCEmJgkCPohLQIhxEQnQdCP6pauBeeEEGIikiDoR02LkyS7lQSbLBAmhJiYJAj6Ud0is4qFEBObBEE/aiQIhBATnARBP2olCIQQE5wEQT+qW5wydFQIMaFJEPSh3eWh1emRFoEQYkKTIOhDcA6BXIRGCDGBSRD0IRAE2XKReiHEBCZB0IdqaREIIUxAgqAPXS0CCQIhxMQlQdCH6pZOYiyK9ATbaO+KEEKMGAmCPtS0OMlMsmGRi9YLISYwCYI++INAuoWEEBObBEEfGjvcpEm3kBBigpMg6ENzh5uU+NjR3g0hhBhREgR9aOrwkBwvy08LISY2CYJeaK1p7nCTLC0CIcQEJ0HQi063D5fXJ11DQogJT4KgF00dbgAJAiHEhCdB0AsJAiGEWUgQALc+soU/rD8Utk2CQAhhFjIkBth0rB631xe2rVmCQAhhEqZvEXi8Phrb3dS3u8O2B1oEyXESBEKIic30QdBgBEBDmytsu3QNCSHMwvRBUG8EQH0vQSDzCIQQE53pg6CuzX/NgVanB6fHG9ze1OHGYbcSIyuPCiEmONMHQWhLoKGtq04gs4qFEGYhQRASBKFfN8mCc0IIkzB9ENS1hrQI2ru+bu6UIBBCmIPpgyC0FVDXrUUgK48KIcxAgqDNFTzzb5CuISGECZk+COranJyWlYhSPVsEEgRCCDOIKgiUUj9SSu1SSu1QSr2qlJpsbFdKqd8ppUqM+xeHfM/NSqlDxr+bo30D0apvc5GVZCclPjbYInB6vHS6ZQlqIYQ5RNsi+LnWer7WeiHwPHC3sf0yoMj4dytwH4BSKh34H2AFsBz4H6VUWpT7EJX6NjcZSTbSE23BeoHMKhZCmElUQaC1bg65mQho4+urgEe03wdAqlIqF7gUWKe1rtdaNwDrgDXR7EM0fD5NQ7uL9EQb6QldQdAss4qFECYS9bAYpdQ9wE1AE3ChsXkKcCLkYWXGtt62R3reW/G3JigoKIh2NyNq7nTj9WnSE+2kJ9oorW8H/NcqBmkRCCHMod8WgVLqNaXUngj/rgLQWt+ltc4HHgVuD3xbhKfSfWzvuVHr+7XWS7XWS7Oysgb2bgYpUBzOSPR3DdVJi0AIYUL9tgi01qsH+Fz/BF7AXwMoA/JD7ssDKoztF3Tb/uYAn3/YBbqC0hNtpCXaaGhzobWWGoEQwlSiHTVUFHLzSqDY+HotcJMxeuhMoElrXQm8AlyilEozisSXGNtGRWBWcXqijYxEGx6fprnTI0EghDCVaGsE9yqlZgI+4DjwBWP7i8DlQAnQDnwaQGtdr5T6EbDZeNwPtdb1Ue7DkAVaBBlJNtISbIB/UpkEgRDCTKIKAq31Nb1s18Btvdz3EPBQNK87XOqNJajTEmykJ/mDoL7dHwQJthhiY0w/304IYQKmPtLVtblItMUQFxtDutEiqG91yaxiIYSpmDoI6ttcwZZAemJXi6BZgkAIYSISBIl2ICQIjBqBXLReCGEWpg6CulYXGUYAJNhisFktwWKxzCEQQpiFqYPA3yLwB4FSigxjvSHpGhJCmIlpg0BrTX1bV4sA/KOHAl1DEgRCCLMwbRC0Oj24vL5giwD88wmqW5y0ubwSBEII0zBtEDS0+SeNpXdrERyrawMgRS5TKYQwCdMGQZ0xmSwjqSsI0hNttHQaK48mSItACGEOpg2CrgXn7MFtoa0DGT4qhDAL0wZB6BLUAWkhX0uNQAhhFqYNgkCLIPTgnyFBIIQwIVMHgc1qIdEWE9wWWIEUJAiEEOZh2iAIzCpWquuiaaGFY5lZLIQwC9MGQX2bM6w4DF0tArvVQlxsTKRvE0KICcfEQeCKEAT+VoB0CwkhzMS0QVDXbXkJAGuMhZT4WOkWEkKYimmDoKndTWqCrcf29ESbtAiEEKZiynUUtNa0uTwk2Xu+/WWFaTKZTAhhKqYMAqfHh09Dgr1nQfhn1y4YhT0SQojRY8quoTanfz2hBBkZJIQQ5gyCdpcXgIQIXUNCCGE2pg6CRJsEgRBCmDII2lxG15BNuoaEEMKUQdAR6BqSIBBCCHMGQaBYnCg1AiGEMGcQBGoE8dIiEEIIcweBFIuFEMK0QWAUiyNMKBNCCLMxaRAYxWKZUCaEEOYMgjaXB5vVgjXGlG9fCCHCmPJI2O70hl2iUgghzMycQeDykiCFYiGEAEwbBB6ZTCaEEAZTBkGbyytBIIQQBlMGQYfLI11DQghhGJYgUEp9UymllVKZxm2llPqdUqpEKbVLKbU45LE3K6UOGf9uHo7XH6w2p5dEmUMghBDAMFyhTCmVD1wMlIZsvgwoMv6tAO4DViil0oH/AZYCGtiqlFqrtW6Idj8Go93lIV5aBEIIAQxPi+DXwLfxH9gDrgIe0X4fAKlKqVzgUmCd1rreOPivA9YMwz4MSrtLho8KIURAVEGglLoSKNda7+x21xTgRMjtMmNbb9tPKRk+KoQQXfo9GiqlXgMmRbjrLuC/gUsifVuEbbqP7ZFe91bgVoCCgoL+dnPAtNYyfFQIIUL0GwRa69WRtiulzgCmATuVUgB5wDal1HL8Z/r5IQ/PAyqM7Rd02/5mL697P3A/wNKlSyOGxVA4PT58WhacE0KIgCF3DWmtd2uts7XWhVrrQvwH+cVa65PAWuAmY/TQmUCT1roSeAW4RCmVppRKw9+aeCX6tzFwwYvSSNeQEEIAwzBqqBcvApcDJUA78GkArXW9UupHwGbjcT/UWteP0D5EJBelEUKIcMMWBEarIPC1Bm7r5XEPAQ8N1+sOllyURgghwpluZnGbXJRGCCHCmC4IOuSiNEIIEcZ0QRAsFtula0gIIcCEQRC8TKUUi4UQAjB1EEiLQAghwJRBIMViIYQIZbogaHNKsVgIIUKZLgja3R5sVgvWGNO9dSGEiMh0R8N2pyxBLYQQocwXBLIEtRBChDFhEMgS1EIIEcp0QdDm8pIgk8mEECLIdEHQ4fLIiCEhhAhhuiBoc3pJlDkEQggRZLog8NcIpGtICCECTBgEXikWCyFECJMGgbQIhBAiwFRBoLWmzeWRGoEQQoQwVRA4PT60lusVCyFEKFMFQfCiNNI1JIQQQaYKArkojRBC9GTSIJAWgRBCBJgqCNrkojRCCNGDqYKgw2gRSI1ACCG6mCoIAsViqREIIUQXUwWBFIuFEKInUwZBoixDLYQQQSYLAn/XkEwoE0KILqYKgjan0TUk1yMQQoggUwVBu9uD3WrBGmOqty2EEH0y1RGx3SlLUAshRHfmCgJZgloIIXowWRDIEtRCCNGdqYKgzeUlXloEQggRxlRB0OHykCg1AiGECGOqIGiTYrEQQvQQVRAopb6vlCpXSu0w/l0ect+dSqkSpdQBpdSlIdvXGNtKlFJ3RPP6g9Xu8kixWAghuhmOo+Kvtda/CN2glJoDXAfMBSYDrymlZhh3/xG4GCgDNiul1mqt9w3DfvSr3eWVYrEQQnQzUqfHVwGPa62dwFGlVAmw3LivRGt9BEAp9bjx2FMWBPGx0iIQQohQw3FUvF0pdROwBfiG1roBmAJ8EPKYMmMbwIlu21cMwz5E1Nju4mN/fj94u9XpkRqBEEJ0028QKKVeAyZFuOsu4D7gR4A2/v8l8BlARXi8JnJNQvfyurcCtwIUFBT0t5sRWSyKopyk4O2ZkxxcfkbukJ5LCCEmqn6DQGu9eiBPpJT6K/C8cbMMyA+5Ow+oML7ubXv3170fuB9g6dKlEcOiP8lxsfzpk0uG8q1CCGEa0Y4aCj29vhrYY3y9FrhOKWVXSk0DioBNwGagSCk1TSllw19QXhvNPgghhIhOtDWCnymlFuLv3jkGfB5Aa71XKfUE/iKwB7hNa+0FUErdDrwCxAAPaa33RrkPQgghoqC0HlKvyym1dOlSvWXLltHeDSGEGFeUUlu11kv7e5ypZhYLIYToSYJACCFMToJACCFMToJACCFMToJACCFMblyMGlJK1QDHo3iKTKB2mHZnvDDjewZzvm8zvmcw5/se7HueqrXO6u9B4yIIoqWU2jKQIVQTiRnfM5jzfZvxPYM53/dIvWfpGhJCCJOTIBBCCJMzSxDcP9o7MArM+J7BnO/bjO8ZzPm+R+Q9m6JGIIQQondmaREIIYToxYQOAqXUGqXUAaVUiVLqjtHen5GilMpXSr2hlNqvlNqrlPqKsT1dKbVOKXXI+D9ttPd1uCmlYpRS25VSzxu3pymlNhrv+V/GcucTilIqVSn1lFKq2PjMz5ron7VS6mvG7/YepdRjSqm4ifhZK6UeUkpVK6X2hGyL+Nkqv98Zx7ddSqnFQ33dCRsESqkY4I/AZcAc4Hql1JzR3asR48F/mdDZwJnAbcZ7vQN4XWtdBLxu3J5ovgLsD7n9U+DXxntuAD47Kns1sn4LvKy1ngUswP/+J+xnrZSaAnwZWKq1nod/CfvrmJif9cPAmm7bevtsL8N/rZci/FdzvG+oLzphgwBYDpRorY9orV3A48BVo7xPI0JrXam13mZ83YL/wDAF//v9u/GwvwMfGZ09HBlKqTzgCuAB47YCVgFPGQ+ZiO85GTgPeBBAa+3SWjcywT9r/NdOiVdKWYEEoJIJ+Flrrd8G6rtt7u2zvQp4RPt9AKR2u1jYgE3kIJgCnAi5XWZsm9CUUoXAImAjkKO1rgR/WADZo7dnI+I3wLcBn3E7A2jUWnuM2xPxMz8NqAH+ZnSJPaCUSmQCf9Za63LgF0Ap/gBoArYy8T/rgN4+22E7xk3kIFARtk3oIVJKqSTg38BXtdbNo70/I0kp9SGgWmu9NXRzhIdOtM/cCiwG7tNaLwLamEDdQJEYfeJXAdOAyUAi/m6R7ibaZ92fYft9n8hBUAbkh9zOAypGaV9GnFIqFn8IPKq1ftrYXBVoKhr/V4/W/o2As4ErlVLH8Hf7rcLfQkg1ug9gYn7mZUCZ1nqjcfsp/MEwkT/r1cBRrXWN1toNPA2sZOJ/1gG9fbbDdoybyEGwGSgyRhbY8BeX1o7yPo0Io2/8QWC/1vpXIXetBW42vr4ZeO5U79tI0VrfqbXO01oX4v9s12utPwm8AVxrPGxCvWcArfVJ4IRSaqax6SL81wafsJ81/i6hM5VSCcbveuA9T+jPOkRvn+1a4CZj9NCZQFOgC2nQtNYT9h9wOXAQOAzcNdr7M4Lv8xz8TcJdwA7j3+X4+8xfBw4Z/6eP9r6O0Pu/AHje+Po0YBNQAjwJ2Ed7/0bg/S4Ethif97NA2kT/rIEfAMXAHuAfgH0iftbAY/jrIG78Z/yf7e2zxd819Efj+LYb/6iqIb2uzCwWQgiTm8hdQ0IIIQZAgkAIIUxOgkAIIUxOgkAIIUxOgkAIIUxOgkAIIUxOgkAIIUxOgkAIIUzu/wN0byUfuFDjpQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "actor_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.001}\n",
    "critic_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.001}\n",
    "agent = ElibilityTraceActorCriticAgent(env, actor_kwargs=actor_kwargs,\n",
    "        critic_kwargs=critic_kwargs)\n",
    "\n",
    "# 训练\n",
    "episodes = 100\n",
    "episode_rewards = []\n",
    "for episode in range(episodes):\n",
    "    episode_reward = play_qlearning(env, agent, train=True)\n",
    "    episode_rewards.append(episode_reward)\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "# 测试\n",
    "episode_rewards = [play_qlearning(env, agent) for _ in range(100)]\n",
    "print('平均回合奖励 = {} / {} = {}'.format(sum(episode_rewards),\n",
    "        len(episode_rewards), np.mean(episode_rewards)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TMa_HS0MwsII"
   },
   "source": [
    "### 邻近策略优化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PPOReplayer:\n",
    "    def __init__(self):\n",
    "        self.memory = pd.DataFrame()\n",
    "    \n",
    "    def store(self, df):\n",
    "        self.memory = pd.concat([self.memory, df], ignore_index=True)\n",
    "        \n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.memory.shape[0], size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ufip7qI6wrl6"
   },
   "outputs": [],
   "source": [
    "class PPOAgent(QActorCriticAgent):\n",
    "    def __init__(self, env, actor_kwargs, critic_kwargs, clip_ratio=0.1,\n",
    "            gamma=0.99, lambd=0.99, min_trajectory_length=1000,\n",
    "            batches=1, batch_size=64):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = gamma\n",
    "        self.lambd = lambd\n",
    "        self.clip_ratio = clip_ratio\n",
    "        self.min_trajectory_length = min_trajectory_length\n",
    "        self.batches = batches\n",
    "        self.batch_size = batch_size\n",
    "        \n",
    "        self.trajectory = []\n",
    "        self.replayer = PPOReplayer()\n",
    "        \n",
    "        self.actor_net = self.build_network(output_size=self.action_n,\n",
    "                output_activation=tf.nn.softmax,\n",
    "                **actor_kwargs)\n",
    "        self.critic_net = self.build_network(output_size=1,\n",
    "                **critic_kwargs)\n",
    "        \n",
    "    def learn(self, observation, action, reward, done):\n",
    "        self.trajectory.append((observation, action, reward))\n",
    "        \n",
    "        if done:\n",
    "            df = pd.DataFrame(self.trajectory, columns=['observation',\n",
    "                    'action', 'reward'])\n",
    "            observations = np.stack(df['observation'])\n",
    "            df['v'] = self.critic_net.predict(observations)\n",
    "            pis = self.actor_net.predict(observations)\n",
    "            df['pi'] = [pi[action] for pi, action in zip(pis,\n",
    "                    df['action'])]\n",
    "            \n",
    "            df['next_v'] = df['v'].shift(-1).fillna(0.)\n",
    "            df['u'] = df['reward'] + self.gamma * df['next_v']\n",
    "            df['delta'] = df['u'] - df['v']\n",
    "            df['return'] = df['reward']\n",
    "            df['advantage'] = df['delta']\n",
    "            for i in df.index[-2::-1]:\n",
    "                df.loc[i, 'return'] += self.gamma * df.loc[i + 1, 'return']\n",
    "                df.loc[i, 'advantage'] += self.gamma * self.lambd * \\\n",
    "                        df.loc[i + 1, 'advantage']\n",
    "            fields = ['observation', 'action', 'pi', 'advantage', 'return']\n",
    "            self.replayer.store(df[fields])\n",
    "            self.trajectory = []\n",
    "            \n",
    "            if len(self.replayer.memory) > self.min_trajectory_length:\n",
    "                for batch in range(self.batches):\n",
    "                    observations, actions, pis, advantages, returns = \\\n",
    "                            self.replayer.sample(size=self.batch_size)\n",
    "\n",
    "                    # 训练执行者\n",
    "                    s_tensor = tf.convert_to_tensor(observations,\n",
    "                            dtype=tf.float32)\n",
    "                    gather_tensor = tf.convert_to_tensor([(i, a) for i, a\n",
    "                            in enumerate(actions)], dtype=tf.int32)\n",
    "                    pi_old_tensor = tf.convert_to_tensor(pis,\n",
    "                            dtype=tf.float32)\n",
    "                    advantage_tensor = tf.convert_to_tensor(advantages,\n",
    "                            dtype=tf.float32)\n",
    "                    with tf.GradientTape() as tape:\n",
    "                        all_pi_tensor = self.actor_net(s_tensor)\n",
    "                        pi_tensor = tf.gather_nd(all_pi_tensor,\n",
    "                                gather_tensor)\n",
    "                        surrogate_advantage_tensor = (pi_tensor /\n",
    "                                pi_old_tensor) * advantage_tensor\n",
    "                        clip_times_advantage_tensor = self.clip_ratio * \\\n",
    "                                surrogate_advantage_tensor\n",
    "                        max_surrogate_advantage_tensor = advantage_tensor + \\\n",
    "                                tf.where(advantage_tensor > 0.,\n",
    "                                clip_times_advantage_tensor,\n",
    "                                -clip_times_advantage_tensor)\n",
    "                        clipped_surrogate_advantage_tensor = tf.minimum(\n",
    "                                surrogate_advantage_tensor,\n",
    "                                max_surrogate_advantage_tensor)\n",
    "                        loss_tensor = -tf.reduce_mean(\n",
    "                                clipped_surrogate_advantage_tensor)\n",
    "                    actor_grads = tape.gradient(loss_tensor,\n",
    "                            self.actor_net.variables)\n",
    "                    self.actor_net.optimizer.apply_gradients(\n",
    "                            zip(actor_grads, self.actor_net.variables))\n",
    "        \n",
    "                    # 训练评论者\n",
    "                    self.critic_net.fit(observations, returns, verbose=0)\n",
    "                    \n",
    "                self.replayer = PPOReplayer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_montecarlo(env, agent, render=False, train=False):\n",
    "    observation = env.reset()\n",
    "    episode_reward = 0.\n",
    "    while True:\n",
    "        if render:\n",
    "            env.render()\n",
    "        action = agent.decide(observation)\n",
    "        next_observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        if train:\n",
    "            agent.learn(observation, action, reward, done)\n",
    "        if done:\n",
    "            break\n",
    "        observation = next_observation\n",
    "    return episode_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sumtZ4R8xPxH",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均回合奖励 = -12276.0 / 100 = -122.76\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsvXm8ZGV57/t711Crxj0PPU/QA80MTQMiiogMJkc0h3tCThJMjJKbqzeJx3MSifeeJHo58YYkRm+MHjSehGiCJFFDlIiAIirS0CDQ0EB30+Pu3r3noeY1vfePtd631qpp195Ve+hdz/fz4cPeVavWWlW9633e3zMyzjkIgiCI9kVZ7hsgCIIglhcyBARBEG0OGQKCIIg2hwwBQRBEm0OGgCAIos0hQ0AQBNHmkCEgCIJoc8gQEARBtDlkCAiCINocbblvoBH6+vr4li1blvs2CIIgzimef/75cc55/1zHnROGYMuWLdi/f/9y3wZBEMQ5BWPsRCPHkWuIIAiizSFDQBAE0eaQISAIgmhzyBAQBEG0OWQICIIg2hwyBARBEG0OGQKCIIg255yoIyAIgmg3JrMmnnhtBJbD8Z+v3rSo1yJDQBAEscI4MprGuz/7Y5iOi8s3dS26IWjKNcQY+98YY68yxlzG2J6y5+5hjB1hjL3BGLsl8Pit/mNHGGMfb+b6BHGu88RrI3jp1PRy38aK4qHnTmFoKrfctwEAePTVs3jh5NSSX/f1s2mYjosv3bUH3/ittyz69ZqNEbwC4BcAPBV8kDG2G8CdAC4EcCuAv2aMqYwxFcDnAdwGYDeAX/KPJYi25JPfPoj/+dSby30bK4ai7eD3/uVlfG3fyWW5ft508KWnjuJnJ6dgOy7+60Mv4bOPH17y+xieLgAA9m7tAWNs0a/XlCHgnL/GOX+jylO3A3iQc17knB8DcATAXv+/I5zzo5xzE8CD/rEE0ZakCzZyprPct7FiKFguAODMdH7Jr31sPItb/vIp3PvIa/iTR17Hy6dnkC7aODm5MHXyyukZ/O1PjgEAjoxm8MUfvgnOOU5N5vC5Jw6Dc17ztcMzBcQjKjqiS+O9X6ysofUATgV+H/Ifq/U4QbQlmYKNgkWGQFD0P4tGDMFouoDf/sef4e4H9uPfXjrT9LX/Yd8JnJ0p4LaL1uC5E5N4+EXvnKcmc7Add97n+7unj+OT3z4Ix+X45+eH8Ol/fx3jGRP//PwQ/uKxQxjPmDVfe3Y2jzWd0SVRA0ADhoAx9jhj7JUq/9XbyVe7e17n8WrXvZsxtp8xtn9sbGyu2ySIFcPzJybxV9+f251QsByYjou8Nf9FZrUxPJOH7bgBReC5Rj77+GE8f2Ky6mt+fHgcD790Bj88NIa/ffo4AGC2YGE6V3uBrce+Y5O4bFMXPnLj+eAc+OozXuNO2+UYninM+3zHxrNwOTCRKWI07b3+zbEM3hzLAEDdDcDwTAFrO6MLeBcLY05DwDm/iXN+UZX//rXOy4YAbAz8vgHAmTqPV7vu/ZzzPZzzPf39c7bTJogVw8MvnsFnHq8v/QEgU7QBAIVV5hrinMvFrhEmsyZuuO9JPPzSGRRt77M4O1vATM7CZx4/hEcOnK36OrE4v2PnAEZmvZ9/759exocemH/L+kzRxiunZ3D11h7sXtuBDd0x2C7HJRs6AQAnJubvHjo6ngUAjMwWMZYuAvBcREdGvc9GvFcA+PKPjuKRA8Py97MzBazpiM37mgtlsVxDDwO4kzFmMMa2AtgO4FkAzwHYzhjbyhiLwAsoP7xI90AQy0Km6MBxuVzoax5X8A2B3VpDULAcHBpJt/Sc8+FHh8fxzj//IY42aAxeG55F0XYxmi5KReC4HD95cxwAarplzs4U0BXXsbkvjtHZIjjnODyaxgsnp5Ez63/25ew/PgmXA1dv7QVjDLdeuAYA8Et7vbTNE5PZeZ1vOmdiMuspk9F0QRqqwyNpaSDyZul9felHR/H15zyvue14n8WKUgT1YIy9jzE2BOBaAN9hjD0KAJzzVwE8BOAggO8C+DDn3OGc2wA+AuBRAK8BeMg/liBWDWIRmslbdY8ThiLfYkXwrZ+dxq1/+RSOj89v8Voo937nIL7zcmk3e3zCu+7ZBt0prw3PAvA+h6BRfPKNUQCA5VZXVsMzBazpiGIwFYXpuJjKWRieKcBxOV4empnXe3j22CQ0heGKzV0AgPe/ZQt+5ZpNeN/l6xHRFKkIRmcL+NW/2YfTc8QwjgY++5HZIkZ9RfDkoTGYtmcAxHt1XI7xjCmPGc+YcFyONeeKIeCcf5NzvoFzbnDOBznntwSeu5dzfh7nfCfn/N8Djz/COd/hP3dvM9cniJVI1l/Yp3P1DcFswXs+3+Jg8XimCJcDDwcCqJ/+99fxj88uTkrmPz0/hIdfOi1/FwZAvL+5eOOsp14KloNiIF7y5BtebNCyayiC2TzWdkYx2OEtmIdH0jID6/kTtXP/H331LO76yrN4eahUv7Hv2CQu3tCJeMTL0tnYE8f/896LEdVVbOqJ44Rv3H7y5jh+dHgcD/gxiXJG0wUcPDOLo2MlQzA0lZN/C0EXk4gRTGa9hV+4j4ZnPCNzzigCgiAqyfo7/dm5FIHvGio2GSzed3QCMwGjIwzRt148LeMU//bSGTz6anVfe7PkTSe0Qx6Z9Ra0uRSR4I2RkiEIBlDFDtmuoQjOzhSwpjOGwQ4DAPCzQGHeC3UMwTdeGMJTh8bw3s//BN995SyyRRsvD01j79aeqsdv7onLBfzQiOfu+ufnh+TO/sREFq+c9hTIZx8/jDu++DQODE1DUxg6YzpeOeMpnk098dB5hRtMuI0mskXYjisN6drOcz9GQBCrFs45HjkwDKuG71oYgkZdQ6bjLig9EfAWz1/+8j78r6ePycdy/nmPjmXxqr8IZYq29Fm3EsflKNouTk+VDIHIkJnNz+2nd1wu4xl5y6kaLzGrfDZF28F4xgwpghdPeoZgx2ASz5+cqhmsf+X0LG7cNYB1XTE8tP8Unj0+CcvhuO68vqrHb+qN4+RkzotBjGSgqwwTWROPHRyB7bj4wN8+h9/9+osAgLF0ETnTwYPPncKmnjjWdcWkkXjLeb2h8wolKJQA5546GJaGgBQBQaxYDg7P4v/42gt44rXRqs9n5xkjAIBCDffHXMzkLdguD8UDsqaDzpgOXWX4zoFhcO4Frifq5K0vFLGDn8pZ0gCKHW7w/T93fBLPHa9MAz05mZM744LlSnWUNEqFVNWM5KivOtZ0RjHgK4IXfUXw85esw3TOCvnpBZNZE6en87hmWw9uumAQT785jh+8PoqIquCqLdUVwZbeBHKmg7FMEYdH07jpgkGs74rh8z84gr/76Qm8OZbFeMa7n2n/PRdtF9v6ExhIGdIAX+sbgvVdsdBnJz4vwFNBZ2cLMDQFXXG96v0sBmQICMJnJm/hE988gBvu+0Fd/7Zw6ZydqR4wzBX9GMEchiBdCBiCBcYJhO85WP2aM20MpAys64rhzHQeRduF4/KWKYLgTjtYFS3cQ+WuoX976Qx+6f5n8H9/65WKc71x1lMsqsJCimBbf0IeYzmVO/vgrtnQVHTHdZydLUBTGG7cNQAAeH24MnPqgL87v2h9J27Y2Y+C5eLB507hys3diEXUqu93+2ASAPDM0UmcnMxh55oU/ug9F+LwaBqf+vZB+V4dl2MmZ0FTmP8ekhhIGfI8e7f2QFcZLl7vpaSK4jnhAvN+LsgagqUqJgPIEBCE5K6/2Yev7TuJ4xO5kKujHCHpRwJf4CCZGq6hxw+O4LpPf1/unIOGYKGZQ6J46uRk6X6zRQdxQ0MqqiFdsOV18pazoOscHcvIyt1/fn4Ib/n096X7J2jATk/lUbAc+b5nCxaOjKbx2w/+DIx52URumb//9bNpMAZsH0iGgsXb+jxD0JOIVHXBlQdUhXtosCOKdf6OW9xjkFcChuCabb2I6gpM28Vbt1d3CwHAns09SEU1fOHJN8E5sGMwhXftHsSX7tqDzb1x/NzFa8G5FxOazpu4+cJB7BhM4i3n9cr70hSGwVQU99+1Bx991w7/s3PlfYo1fyxdxJnpvHzdUkGGgFiR2I5b4eMVX5LyxaQV5E0HLw3NYM/mbgD13TryCzxbaQhsx0XRd/OUn+ONkTROT+dlumSmWHq+uMBaAqE6xjNFmbaaLdpIRFSkDB3pQsllAwCTC6i6feCnJ/C7X38Rpu1i//FJDM8U8Ml/83bCwYynoel8yM0xk7dwaCQDzoH/tGcjCpaLkbLF+ZXTM9jal0BPIuIFi/3P4abdg7h6aw+29SWkIXBcLv/tRUB1jR9QHfAXzrWdUXTFdGgKC+20BQeGZrClN46OqI6oruItflyg3H8fJKIpuOmCQfnvtn3AUwg37BzAD//bO/Cu3YMAgKmciemchY09cXzvo2/HDTsHpNuqL2lAURjesXNAqh25oZgtYktvQv586GxaqpClggwB0TJmCxb+4rFDNYOo8+Gmv/ihbBsAAD94fRRX3fs43vLp7+OTvhxvJcd8f/Ie309cL+NHLNrVdpzZwI57pix9VLibDgpDEFIEjX9mjsvxuScO49RkLnSNU74qyJoO4pGSIgjGIiYzprfznofhmcp56Y2npnI4PpEFY8C3Xx7GDw+NhV1DU3npFtIUhtm8JQOhIiPn2HgWTx8ZxzdeGALnHM+fmMKVm7oR1VUUrFKLiZ+7eC2+/pvXwtAV2L5r6H1//RN89gmvdcfwTAEpQ5OxhEHfBbO2KwZFYehPGfLaQQ6cnsGFvmsG8ArGrju/V7pranHLhd5ir6sMW/oSoec6fV/+2ZkCiraLrlhEPidcQyKzyTuHAk1hUk2NpovY0B1DZ0zHz05OIV20sXtt/ftpNWQIiJbxo0Pj+NwTh6UfdqEULAfHJ3Ihv7dYqPuShszpbiXi/Jdt9AqK6ikC4V6ppgiCFa3l5xAL/8FAJo885zxiBN979Sz+4rFD+PbLw5gK7PDF55UzbSQMFamoXmEIJrJFfOiB/fiFv366rpuoaDvydSIOcXw8i5MTOfzcxWsRj6j4weujoXOcDiiCrX0JzPiGQFUYrtjU7Z8jhz9/7BD+4JsHcHB4FlM5y/PP6yryvoGKaIr0j2uKIgvKTk3mZBaUlzpacp+In9f5/x9IGRWKYCZv4fR0HhetKy2y79o9iK998Bpoav2l8G07+mFoCrb2JaCXHdsd9xb+Y/7fZTDIK5RKfyrs6hGGDwDGZgsYSEUxkDLw06MTAIDd6zrq3k+rIUNAtAyRLTNVIyiZM+2KXXI1xOJWDGTSiEX1vP5EyLcu+PcDw/jmz4Yaus/RdKFCtYh2CJdu9BaJ2SrXEJR2clUUQbGOIfCfEy6G2YINXWWhc84F5xz/86mj8vrBgPQp3xBkiyVFMFuwQspjMmvi4JlZvHpmFvd84+UK95vrcjx+cAQ3/tkPcccXngZQikO8MZLG8GwB5w8k0RHVkS3ayFveuRMRFUNTOWkIdgymMJO3MZYuojcRwfquGCKagkMjaRwYmkHBcvGXfp//Kzd3w9AV5E0vRhDVSsuSrjJZUGbarjz/8GzYEARdQwDQnzIwOhv+9xEdTcvz+RshHtHw4XecjzuvqpwU1u0v/Mf8IrKuWMAQ+IpgIKAIACCqKyjYDlyXYzRdxECHgf6UgYLlQmHAzsHUvO+xGcgQEC1D5K/Xyk7544cP4tf/9tk5zzOV9RY3s8wQpAwNnTG9qiH4yk+O4UtPHat4vJyC5eDGP/shvuZ3lhQcHc9iXWcUA/7OrW6MwL+vqZxV4WLJ+hlDqahWUxG8fjYN23GRKdjoS3oLRKOK4PkTUzJNcixdxHTOQl8ygqShhRRB0vB62WeKNtKBWMSJiRwmsiY29sTwrRfPyHMBXmuKq//kCXzwgf04O1vAm2MZcM6lsfnRoXFwDmzujSMZ1ZA1benSOn8g6buGCojqCjb0xDBbsDCWKaI/5fnHN/fE8ciBYVkX8NjBEXRENZzXn0RMV1G0vYKyqF7K3tFVBbbrHW85XBqCM9P5UJ590DUEeDtwkdIpKMUVFhaI/e13bscH3rq14vEuXxEc94vOOgOKoD9lIB5RsbU37E6K6ioKpoOpnAnb5RhMGdJobO1L1MxgWizIEBAtIxcoma+G14J3brdONUUwm7fQEdN9d0flIj2btxuqZD0ymkGmaMsvreDoWAbb+pNQFebtpBtwDQGo8EMLRbC+K1bRDlkYsKLt4th4FpmijX7/y1+wHPzd08fx9edqt4HgnOO+R99Ad1zHxes7MZouYiZvoisewcaeOE5N5uC6HDkZI9DBeSmdEyjl2r/v8g0AEGqv/A/PnkREVfDZOy/Dx27eAcvhmC3YUuHt99tBb+5NIGFoyBQdacDOH0hhNF3EqUkv46UzpsO0XZyazMn3uKUvId01b9vhdRS+YnM3FIUhqqueIrBdGHppWdJUBZbDwTmH6bgyKD6WLmJDd2lnv3drD26/bB2u9mMRAykDE1kzVIOwWIVaHVENqsJkj6VgjMDQVHz3d96GX712c+g1UV1FwXbkv81AR1R+TrvXLW18ACBDQDTIc8cn8fF/qXQlBBH587UyU7yFy5rTDSIMiRnYbc/kLXTGdC8AWqWr50y+sT70h0e93PKJgLHinOPoeBZb/SBgR1SvawiC1a/lfmgRLF7XFUO6aIcynNJFGxt7vB3rweFZZIq23AUWLAf/sO8kvvSj2qrmof2nsO/YJH7/1l3Y1BPHuK8IumI6NnbHcHIyJxdmL0bgBVLFTjhpaNIQXL7Ji4UEjfaR0Qyu396H2y9bj3V+Ns5YuiDdZCKff3NPHElDRaZgIe+7Ay/zz/fYayMYTEXREfV2xccnsuhPlna6gKcoftkfxn6lHzuI6SoKtuspAi2oCBgsx5XXdjlkQzlRmAV4u/LP3nm53J33pwxwHv53PjuTh8IgF9xWwRhDV0zHSX9zUV4Itqk3HlI5gO8aslzpXhxIGVKN7l67tPEBgAwB0SA/OjSGB587Vbe1cr0YAedc/tFXy+YIUitG0BnTpbujPIV0Jm8hazohd1KQ509M+u2ZvVjARMBtMJ4xkS7YMq2vM6bXVRfB3kDlfmihCNZ2RsF5uFYgU7Rw6YYuRFQFB8/Mhl1DpoPpvIk3xzKhOEPwvPd+5zXs3dqD/7Rno+cDF4YgrmNTj9cGQbxWKALAc6Mw5i2c4n1dtsFbuMW/1USmiMmsifP91MjepLegCgUnFrekoaEn4bmisgFF8N7L1uGTt1+ImK5i55oUOn0/ueVwufBu7vV28Fdu6sYNO/vx69dtwS9c6SmTqK7I1t0h15Ci+Iag9JmLhnLru2v34hEGNhjQH54poD9lVAR7W0FXXJcur0YqgqOaioLlyCll/SlDxhEuWLu08QGADAHRICJzo15HTeEyqeYayhTtiiZbtSgpgkpDkIxq4LxkdMRxYkGqtoBPZk3c8cWf4m9+fAyH/b42wXYLIlC8rd9bBDtiXpDVdlw8+cYoOOc4M53HHV94GqPpAvKmI4O8lYrAuy9R1DSdL10nU7DRk4jggnUdeObYJEzHlYtk3nIxnbPAeSmYHGR4xtuZ//LVm6AoDAMdBjJFG8MzeXTGIljTGUXRdqX7I6QIZgtIRjS5uPcmIuhORJAyNLljFsNStvtByt6Ed19iwMylvuHY1BMHY8x3DZXmLccjGu66dgue+8RN+L9+/gJ0xMJ+cgDST3755m4Ymoo//A8Xyl29WPyncxaMYLBYY7AdHvpbEA3lgoqgHBE8Dgb0z84WZN1BqxFKJKIqiOlz+/ejfpaU+HvtikVww44B/Obbt+GabbVrGhYLMgREQwhfaz1DkK1jCMJl9HMogmxtRSB2ucGddnDxn8lXu3YBnANPvDaCw/6CN5Et3YNIHRXVrEIRPP7aCH7tfz2Hn52axo8Oj2H/iSkcPDOLgu1gTWcUqsIqjFowRhC8N9HvJ2louHJTt2yBLBTBdN6U77da+q0ISosFU7hbpnxFIM4jAsYiawjwxj4mo95OHijtzLsTEam+xOciFEGfbzSEgRCupC193muThh8strx0T9VvqxCLqDA0VSqC4Hvcs6UHv/PO7XjPpesq3p80BHkzpAg0RYFZpgheODkFhdUP+grjE1SfwzMFrF2kil2ROdQZ1xtqDSHSR4ULMhnV0BnXcc9tF1S4kZYCMgREQwgf7VQdP7zwF09VMRbBL2S5O6Uc8fpyRdARKy1umRppmtUMldj9/+zUNE5O5hBRFdkDHij5kcXi0RnTMZu3Zevhl05Ny9z/nOlltsR1Df1Jo6KWQGQNiUVK3FvR9vzcyaiGKzd3Q4RaOmIaDE0JDXGpbgi8z0LslgcCC1pXTJe7fWEIEgHX0HimiIShoVcaglL7hsmAIkhEVJmH350Iu4ZEfYV4bdLQkCnYyJsO4lUyXDqrKIKIpuCj79oRek4Qq6EIIppXUBbsQDqVs7C2M1bXxSMMZXDTUV570EqEIuiq8t6qEdUVFC0HswUvG04Y0uVCm/sQgoDckdUzBGIRnMhU7viDX8haPXoE4hpykpPlZZOEFUFQBdQ3BCKNUCy+l23qwrPHJjGdM9GbNJAu2IioityJdUQ9RSDyzg+cnpE5+sLFFdUVGHpl0VK2aCMeUWWR0ctDM5jMmrKVQcrQ5BQsAEgaOmIRVbp0VIXJfjhBRIDd8AOpwWZmXXFdunJEwDLup4+WrqPJxV3k0fckItJ1cmQ0g/MHknI3q6sKOmM6jvqKYEtvAp/5xUtx9VbPbZEwNNgux3TOquoKCV67keCsSJdMF+wyReAFi8tjP/XcQoBnQLrjOsbSRanGMkV70Vo7C0XQaMdQTxE4/gZn6bqM1oIUASGplxEkyvyncxa+//oIbrjvBxXZP6KqdrZgVxRsCRWQNLSqFblBJqVryPHP5y3unTFdthQIFnwFM3yqdfwUikDsXK/1fbBCCaQLFpKBhaszpiPvVzcDXn+a1/xOll4RlQNDVzGQilaMYxTtHcSCcN+jb+B3HnxRLripqI61nTG5kCUNDTFdlS6myzd24chopmLmrlAEUT+1Mri4dsYj0pUjFEHSKCkC8btQBMK90x2PYDIjXENpnDcQ7m/Tm4zIDK3ueATvu3yDjH2If4fxTLG6IaiiCOoRDaSMBtNHvToCXjGToF6gOHjdHx8Zx97/8YScB7zYiqAzkDpaj6iuIG85mM3bZAiI5cVxOe5/6k0vYyVn4tI//h6+//pI1WMtt6QInj8xheMTuYpFMNh3pnxnPpYpIqIqOK8/UbUiN8hUWbBYLPQdftYQUDtGUC2FdCJbhKYw3Lx7EBFNwZV+YzmhFNIFW7qcxHUA4HW/RfJhv/YA8AxB0XIQ01WcN5DA0fFMaLcqirk6YzoUBin5hYtFLKBX+PeQimqI6iVFcP32frjcG54SRGQqCUXQE4/Ic3fFdLnbL8UIVER1RbZEThqadCdt7StlBk3mTMwWLIzMFrF9IJyt0uerDIUh9PkE38dYuli1+ElXFcQjKgxNQcqY2/EQVAFGWfooUPrbEu95LkUAAAOpKI6NZzGWLuKzfhXzYk39EgqwUUUQEzGCghVST8sFGYI25pXTM/gfj7yOHx4aw5lpLyvlyzXy2K2AIhBFMMGAK+B9WcXOrjxgPDZb9FPkonMrgrL0UbHQB11DmZrB4uqKoCcRwT3vvgAPfGCv3BUKpZAphg2B8GGPzBaxpiy4mCk60jV04bpOWE5pwhYgXEPe4v7l9+/BfXdcAqAUdBXK46otniHoTkQQ1VVpTN61exAKA350eCx0XaGOxG5ZUZhUAV1xHbrqDTIR7ZkTEQ2MMfm+EoaGG3cN4Iu/ciUu3eAVLHXHIyhYLl7x8/K3V1EE4vNQynzYCWEIaigC8br+lNFw8LT0c7igDCjVqKzr8v49GlEE12/vw9t39OODb90qlc2iu4YajhF4BWWz5BoilhuR6pguWHLH+/SbE6FpVwI7ECMQboyxdHixz5m2rPYsNwSjac8QDHYYdRVB3nRkmqlZ1RAIRVC5+KcMrUaMwIsFDHZEcc22XukimZCKwELKKH0Zg8HMm/2ukwrz+ukI11BUV3GR3xhMBJIBL04idss37hrEJf6iK9IwxXO/eNVG/P1v7MX6rhhigYVvc28cV27uloPbp3MmOOcBRVA6VhQgiUrW3kQEorwibngLqzCcqagGXVVw60Vr5MLck/Ce23fMqxg+v4YhEG6PIOJ9TOesmu0QOqKlbKa5iIUMQbjFBFByO27o8v6+NjRgCH7z7efh7z6wF79x/VYIO1be86dVdC4gRsC597dZLXi+1JAhaGOEj98LpJUW0Ad9f2qQoCIYraEIsqYjv6DlQeWxdFFWT1br0SMQr+uO61UVQTyiQmGVrqGYrqI/ZVSPEWSLcvcMeAubwkrGKl2wQzGCjljp5ys2daM3EcF5/Ul0xSPIFm0UfNfQlt4EEhEVr54pBXezpi0XYaCU3fOmrwhExa2hqbh+u9dmQSx8usoQj6i4YecADpyewT/tP4UrPvUY9p+YqkgfBUoBY7EI9fqLrqYwRPwFVBjOZBX3TI/v+nn22CQimoKNZc3YRAC62uIW/LxqKYL3XLauaqpoNcKuoXDTOaDkGhLGaktZ7556rO2M4cZdg3Ka2WIgXEOdVYxmNcR7nMgW5d/EckKGoI0RDcMygSlW2/oSchpVENH4azpnyuEi4wFFYPuZHcIQVCoCr6pzoEp+dxDxusGOKEx/OI3oWNoZ83K0k4ZWoQg6Yzo643r1GEHGlCoA8PzMPYkIxgOGoJprCPBcEB+8fht+9drNXsqkbwiiugpFYbhgbYdsjQz4Q2ECi27KDwaL+bnJKv5gsZB2xiJgjOGGnZ6B+P1/eRkuL/W5B8KLZH/K8Pz3/vWEsYtHVLnrD7qGyhGK4IWTU9jWl6hIYZSupyo71mTA2FVLHwWAD7/j/KpN2qoxlyIQ6vV9V6zHwx+5rsJozcV9d1yCr37w6nm9Zj5s7UvgtovW1B1wE0S8R87DG4/lggxBG5MPKQLvi3bNeb04PZ2vyAgSWUNnZwvS/RJUBKLh3PquSteQabuYylkYSEX+/7a3AAAgAElEQVTlCL5g2mW2aMv0TKEIhC+3aLuYyXv3Jnypqage6jckDEFXjdYQE5mi3C0LehNGyDUU3JUFf17XFcNv3XAe7rp2CxKGiqzppY8KX/2F6zpwcHhWtrzIFh0kAgsjYwxrOqPSzZUwKhfNqH+82HnvXtuBgZQh3Tx5y6lIHwWA/3DpOnzo+m3Sfy928CFD5L+XagZI7GKLtisrikOfkf+ZdVfZ5Qav0YpOmaGsoYCxE8FuESOI6Sou2dCF+dLtq7rFIqqr+MKvXNnwNYKGjxQBsayIArB00ZYVsaLh1VDZzF6RvhfsZBls8yu+qMKPHzQE4rj+lCFTCYNFZV948k3c8pdPYTJryteJgK7puJjJW0hEVLk7FJO3BNIQxCMVMYK86SBrOrKqVtCbjGAiY8oc82pZQ6rCZHtjwFv80gUbpuPKL/KF6zuRMx05lCRr2hW774FAQVU114RosiZ23owxfOzmHfitG86T76Foe33qhasEAK47vw/3vPuC0HsCwjv0kmuo8rrCcACVgWLveeHuqFyoQoZAb35HG62hCCJaWBEsRp+g5SD4HilYTCwrUhEUbGQKNhgDdq3xdoanJsNtmu2yPG6FhV1DIpiXMFT0BFoXAJ6KALxxfWJBDi7Yp6fzyJkOvvrMCZk6KpRD0U+xC7prOspaUYvMi85YpWtIqJZgjADwdrsTWRNZ04HLwz70qK4ioilY0xENTa5KGprMNBJf5Av9gPGrZ2bBOffnBYcXRvFeaqUJxiLeNYK++F+8ahN++8btAOBP7nJhaGrdDByxgw8u0mK3mTQqF5tUtFTRWh4oDp6vqiKIBBVB88uIoSlygHsoa0gRwWLvbzWyagxB6X1QsJhYUo6PZzE0VVrgZYygaCNdtJGMaNjk96E5WW4Iyrp9butPYjzoGjJL0r07HgkpAjFaUgw0EdcUiGP/7unj+Oq+k0hFNZkRIxRBcNeU9BXBm2MZnJzIBRSBjtmCLVtHAKUU0eDu1/s9gvFMURqUVJk874zpMlVRkDA0qW7EFK3tAynoKsOrZ2ZQtF24vNIfL+bVVgvYeucqxQiCCBeJN7nLCRVaVaPPN7LBRbpDxggqFYGiMJn2WE0RrO2MImlositrEFVhUnk00mRtLhhj8nOoXkfg/b1EtNWxZIUUwQqoI1j+OyCWjN/7l5fRGdPxpbv2AABy/phBoQiSUa9/TlRXcGoyh/FMEScmsrhyc4/MGhLsXtuBJ98Ylb8L11LC8Kpqg4bg2HgOCgM29sTlDi/o2pnKeXn+E1lvsPrf/NpVslit6JfhB3dNqaiGN8ds3P3AfiQNLRQjADyFIAqshCLoLVMEfckI0gVbqprygqnz+5PYVdYOOGloMmgr/OIRTcGOwRQOnpmV99xd5koRiqCanz54rvLsHG9gi4KC5aXUGnMsgiVFEHQN+XGVKorAu1fPnba5ShZOwtCw7w/eWTMYnDA05EwHsUhrlpFYRPVTc8OVxUDJ9Rh0jZ3LrDTXEBmCNiJdsENtJAr+Lj7tB4uThleEtLHb623///776/jewRG89Ic3w3JcpAxvKExEU3D+QBIPv3QGpu0ioikyWByLeFW1RwOTyI6PZ7GuKyZ3eomIGh6onjHx9h39uHRDJ/Zs6cFF6zvxyIFhAJ4imM1boTmzqaiGM9P5kHESMQLAazMhDIHo916ezy7aER8ZS8tzBvn739hb4YYJLojBL/KF6zrw+GujePa4l48vqoYF0hDUUgR6OEYQJDjUfa6ulKUYQWUqbPn7C77G5bzmTrtatpEgZWheZXGLumUKlRVWBOEYwepRBKX3QYaAWFJsx0UwqaaUNeQVlIkvvRhyMpO35E7f9nvnp4s2BjsMubBOZItY2xmTO7ZERKvI3jk+UZr+BXg740wVRfBr15VSDcXu17QrXUOpqF6hUDpjmgxqenEC73rSNVSmCERWkhhUU+4a0qr4ooOLYnCxumh9Jx7aP4R/ffE0uuM6zi/LHCkZgupfeGkIqgRlY7qKnBjhOMci2JeoVAS3XrQWRduVrafL+djNO2sO85kL8XnUUgzzRWRPhSuLw3UEqzFYTDECYkmx/Xm2grxVqiMIZs5s7Inj0EgawzMF2C6H63JYDkefn/2ypiMqF1ax0AofbjyiojMewWzBguN6s2aPjWdDBUAiHx/witpyVbJ6xM6vaHtD3oM72mQgb174tjvjJdeQCFTP5Cy8cnoGMV0N7ZKBUlaSGFRTa8ccJFkjZVIEjH9yZAJ7t/ZUtGMQMYKawWJRR1AlKBv13SUFy5mzGKojpiGiKSGj1hnTcde1W2oGma/a0oPrzu+re95aCIPTqv759WIEYkOyeoLF3nsUFevLDSmCNsJy3NCIR5E+mil6BWWir86mnjiCsWHTcWG7pWlaAx1RqQjG/OBpaVKV5xryxjR6xiBdsLElpAhKdQAillBhCPwvfNFykTHtUOMysaC+5bw+bO6N4/D3j6AzpsvF/cx0AROZIq7/0x8gZzqy22gQ8V5LimDur0JQEUQDu/NdazrAmFccJNo0B5k7RuBnDdVwDRV8RRCdI1jMGMOX7tpTNfC7GAiF0ypFEKuiCEotJhxoCqswsucq4u+nI9bYIJvFhgxBG2E7PNTaQbiGLIdjMmvKHW951abpDw/viumI6SrWd8VkOqZQBFkzECz2F7SZvCWzbLYEXBMpQ0PGz9YRhqA8RdEITKzivHqR1HXn92Lv1l5879UR7FrTgcFUFLrKMDSVxxsjaeRMB396xyW444oNFZ9FwtDQEdVwys+iKncNVSOYix/cBScMDVv7Ejg6lsXerT0Vr4vqKt65awBXbal8DiileFbryxOPqDJ9tBFf/Nt39M95TKsQn0crCsqAkgEIKgKRXJA17VXjFgJKn9lKKCYDyBC0Fbbrhl1DgZ8ns2YoRgB4stzy58VajgtdVfDVD+7F5t6EXJTEQp83HTDm+faFz3M6Z+HYuLfQBhVBKqrJxnPCjVNLEQhDETQEl2zoxGUbu3DjrkH0pww8+tG3yefWd8Vwaionp4tdu6235i5ybWcMsyNpsAbleb1q2ks3dGEiY+ICvyCvnL/5tatqnvftO/rxlV/bU3VoeVT3AuuW4zbc2XKpEJ9Hq4LF4jzhgjLv3y5vOqsmUAyU3GArob0EQDGCcxLLceWwlvm9jnu57r7fJ2gUgJJ7ZGtfAm/b0Y/3XrZeXs92OHSV4crNPehLGkj4PXREXYLXWsHLOhJBz+m8hePjWS91tLukCMSYQ6C2a0jkzAvFEXTdbOtP4lsfvq7qwJONPXEMTXqGQFeZHKRSDeFKEtlScxF2DYUXv3tu24V//NA1Cxo5qKkKbtw1WPUeYroqO7LOVUew1AhXV6sUgVCBRpWCsmzRWVWKQPEbA66EQDFAhuCc5P6njuLWzzxVd6JYNUR1sHAJiS6aAuEaimgKHvjAXlzj+9ZN24sRlGfSvHV7H7627yQefPYkcqYtfcXCEMzkLRyfyGJDdzy0m0tGtYZjBFIRNJirvqE7hqGpPE5OZrGxO153YRaZQ43K83D1cfizGOiIYve66mqgGWKRQProInXOXCjJyCIpgmCwWBTVWQ4iq6SGQGDoyopxDZEhOAc5M53HmZlCaFxjI1i+EhCGIG85of7s5cHMSCCF03I49LJF9XN3Xo637+jHx79xAC8PzUhDIFI9Z3ImTk3mKlIXU37WEOccU1kTjFWm0IldoRhSUy+fPciG7jgmsiZeG07LKulaCEXQSKC4/B6iS5TpIRRBcQUqgsHOKCKa0rI8+JiugpX1Uwr+za0m1xDg9d5arIlp82V1fbJtghjcIoarN4pUBL5LKG866A8EKMsLnoIZG8HfBbGIij+94xIoDDg4PCtTNDsDweLT0/mKISLJqAbOvfNO5kx0B8YuCgzVW2gnq7iG6iGudWw8i81ztCpeO09DkIzUdg0tFlG98fTRpeZ9l6/HYx99W81Cufky2GFgMBUNuciCf3OryTUEAP/wwWvwX27esdy3AYAMwTlJwc/8mY8hcF0eamss/h/0s5d/oY2yzo/ViqwGUlGZhy4UgaGpiEe8ObzjGbNivqxIO8wUbUxmzYqWDEBAEVQJFtcjmPFUrW1CEFFd3OhCJgbOlHcBXUxiERUF0XRuhSkCXVXm/Iznwwev34Z/+z/fGnpMU6sbhdXAGr+X00pgdX2ybULRmr8hEMPnAU8JWH5K6EAdQxAJND0Dai9+t/tB5Xjg9Z0xHa8NewNbyufLChdUuuAZgvL4ABCIEUjXUGO74aD6qFVNKygpgsZcG7qqIKIpvgtjaQxBXFdlkH+lKYJWE/WnzAUJLv6rzTW0kmjqk2WM3ccYe50x9jJj7JuMsa7Ac/cwxo4wxt5gjN0SePxW/7EjjLGPN3P9c5Gi7eDpI+NNnUPs6IfmYQjsQEuGnFkadBJSBDViBKIKWKsReL3lwkFENCWUgukZAq9qVwyrEYjisEzRxlTWqtrmWFEYdJXJttSN7pz6k4ZUMnMZgvnGCMR9tKqSthGCGTlztZhYjYQMwSpTBCuJZj/ZxwBcxDm/BMAhAPcAAGNsN4A7AVwI4FYAf80YUxljKoDPA7gNwG4Av+Qf2zZ895Wz+M9f3icHwC+EUoyg8XMEDUHBcuQuvzMekQv8XDGCaq4hwNtR33fHJfj1QK+grrguDVa1GAHgtbaYyJoVfYAEEVWB7XIorPHMFMYYNnTHwJgXOK5HytCwpTdetRd/LRKGuqSGoNbAlnZBVZicU0CKYPFoykHFOf9e4NdnANzh/3w7gAc550UAxxhjRwDs9Z87wjk/CgCMsQf9Yw82cx/nEmJ3nS3OL+MnSKFJ11DOdOQiHddVJKMapnNWpWuozBDU25EJ95BABIw1hckWCwJxnXTBwpQfLK6GoavImg4SDeb5Czb1xFGw3DkXTsYYvv+xGzAfL08iosFyFtakbSEEDWA7KgLA25CYtrtqWlCvRFoZqfgAgK/7P6+HZxgEQ/5jAHCq7PGqE6UZY3cDuBsANm3a1MLbXF4sv9NjeffM+bAQQxBUBHkrYAgiKpKGbwhquIZywjU0jy9ilz9kZU1ntCIjSBiC09N5OC6vGiMASoZnvgG1//KunaEJafWYb++apKHJz24paHfXEOClkJpYfcHilcSc3zDG2OMA1lR56hOc83/1j/kEABvA18TLqhzPUd0VVXVF5JzfD+B+ANizZ8/CV80Vhpj01cyuUriGRmYLsvXDXASvlw+4hqK+IYiolfN0xcKTqZM1VAvRErrcLQSUfPIHTs8AQGjWQBBhiOZrCC7e0Dmv4+fDlVu6Q605FpuQImhD1xDgF5WtshYTK405v2Gc85vqPc8Yez+AnwfwTl4qdR0CsDFw2AYAZ/yfaz3eFogh8M0ZAgcRVYHpuBiZLWBDdxzfeGEIgx3Rmi2F7bKuo2Ixi+kqUlGtalZOxXSoeeyehWuoPFAMlFJBXzg5BQDYMVjZYwcoGaJGU0eXgntuu2Dug1pIKEbQpguhaDNBweLFo9msoVsB/D6A93DOg0NuHwZwJ2PMYIxtBbAdwLMAngOwnTG2lTEWgRdQfriZezjXEC6aZl1DW/q8BfbMdAE508bH/ukl/PKX9+Gebxyo2noiOHw+b7rSvRHTPUVQrUWydA0tYCiIaDNRnjoqzuONw8zD0JSKbqfl118pudbLQbDFc7sqAtFaghTB4tHsJ/tXAFIAHmOMvcgY+yIAcM5fBfAQvCDwdwF8mHPucM5tAB8B8CiA1wA85B/bNlitUAS2i219XqbLmek8Do1kwLk3gPwfnz0pxzOGrxtIH7XsUIzgxl0DePdFayteUzIE848RCEWwoUbTN1FUdl5/smY/IIMMAcUIUHJJUoxg8Wg2a+j8Os/dC+DeKo8/AuCRZq57LiMWZHOBhsByXDgux7Z+r6Lz+ERWzhj4uUvW4i8fP4x0waoozLEDWUMFMxAj0FX86rVbql5LTodagCIQxVriPstJRTWMZ4rYMVg7dTOyAl1DS02szdNHgdLfIRmCxaN9v2HLhFAC9gJdQ2In3x2PYNeaFJ4/MYXzB5KIR1Ts9nvhBwfD//0zJ7C+K4rOWCkzJ5g+Wq+FsEwfnaOgrBpXbOrGtz58HS6tEbgVu/ztNeIDQGlASbLBquLVSJTSR6UBINfQ4kGf7BJjN+kaEqmj0YiKvVt78PyJKbx6ehbbB1OyC2Q60JX0S08dxT/tHwrHCAJZQ/XGDDLm9UyXimAeX0TGGC7b2FUz/18agjrFXDJGMI/K39UGuYZKLsnV1oZ6JdGef1nLiDWP9NGJTBGny2oFin7qaFRTcPXWXuRMB8+dmMSuwZRMy0wHhtbkLcefJ1BWWSwMyhz9ayKaImMEutK6PxexuNfKGAJWZtbQUhPMFGrXYDEpgsWHPtklZj4FZfc+8hp+9cv7Qo9JRaCruGprNwBvaPrONSk55CI4p6BgOv7MYe+6CvNdQ6YDQ1PmLKjSVYZsUbSYaN2OLGVodTOGAMoaArxAqXDRtWv6qNiAUIxg8Wjfb9gyMZ+CsomMiaPjWZyZzsuRi/mAIRhIRbGt3xuavmtNSRFkAoZADD4XMYmOmC4rixsZMRjRFOlqamWJ/517N+GKzd11J4iVYgTt/Wcai6gw8277KgKNgsWLDX2yS8x8CsrEov/ssUn5mKgqFtkkV2/tAeApglIPH1tew3a5HDUJeNk6eV8RxBtYWDzX0PyzhuZi79Ye/Mo1m+seQ64hD/Fv3bYxAoVcQ4tNe3/DlgERtDXtuQ2BcAPtOzaJ916+PvSYmJl799vOw+61Hej1J43FdFXGCMSxYtQkAKQMHVnTqyNoZNxicPGfT4uJVkB1BB6xiApVYW27I5YxgjZ9/0tBe3/DlgGxIAeDt7UQmT37jk3Ix4IxAgDY2pfA1r5Srn4qqklFIBSF6ZQUQUfMy9/Pm05DrZ2DX775tJhoBRQj8IjqatuqAaDkkiRFsHjQJ7vEyMriBhSBWMiPjmUxli4C8KqKgZIiKCcV1WQdgcgwCimCqI686WAsU6zZ/jlIcAFaakUgjBC5hpQ2NwQULF5s6JNdYubTYqJgObhwnVckJuIEBTOsCMpJRXXM+q6hfMA1JIPFUR05y8HrZ9Py3PUI7sKWuh+8mNHb7oogFlFX/ZjKemiyspjqCBYLMgRLjGw616Br6KotPYhHVDzru4fE4PrahiDgGjIrXUOpqAbHDyBfuH7uds3BXdhS78gu2dCFPZu7a84raBdiulpTAbYDEaojWHTae6u1DDTqGuKcI2c5SEU1XLm5G/uEIrDmNgRiYE21YLGoPgaAixswBMEv33xaTLSCa7b14p9/6y1Les2VyNt29GNzb/WeTe1AqbKYDMFiQYZgibGcxuoIirYLzr0F/+qtPfiz7x3CVNaU6aO1iotShi5jBGHXkB8s9msNUoaGzXWKuQTBL1+9nH9i8birRlPAdkGkj86nxQkxP8gQLDHCAJhzVBYXAvMCxMSt545PomA50FVWZ5B8yTVUCGQNWdIQeIpg97qOhsY0CkWgq2xec4MJolWIv0FSBIsHGYIlRqSN2nMogmB30Es2dCKiKdh3bBIu53X7AyWjGnKmA9txpXoInk9UHzfiFgJKXz7K2CCWC+GSpL/BxYM+2SXGtBvLGgp2BzU0FZdv7MKzxyZRsOq3Gkj5O/5M0Q4NWc8WHSgMiPsZOI3O9RW7saWODxCEgJrOLT70yS4xIntnrqZz+bKg8EXrO3FkNIOi5dTNICl1ILVDQ9Zzpg1NVXD5pi7cedVG3LBzoKH7LbmG6E+FWB50ChYvOuQaWmIaDRYHYwQAsK4rhrzlYHimULciuCNgCESqKeBNGdMVho6ojk//x0savl9hAFrZeZQg5oMsKNPob3CxIBO7xDRaUCYavYkOoeu7vNGPb45l6o4sFLOA0wVLFp8BQLZoL6gymBQBsdxQr6HFhxTBElMyBHO4hsywIljf5aV6jqaL2NxbO+0z5BoKxAhyRWdBlZkULCaWm9suXgOX87YvLFxMyBAsMXaDw+vLYwTrfEUQfKwaciZB0Q5lDWVNW+ZjzwcKFhPLzdrOGD54/bblvo1VDW3zlhDOecPpoyJGIGYK9yQisvFYfUNQcg2FFIHpLMjPH5ExAvpTIYjVCn27l5CgO8hyOA6emcXeex/HaLpQcWy5a4gxhvX+lLJGFMFsoTx91F6Qe6dUzEOKgCBWK2QIlpBggNhyXBweTWM0XcSR0UzFsXkxiSwwPEaMq6w3u9bQFOgq87KGzDJFsAD3jnQNkSIgiFULfbuXEDukCFy565/MmhXHit18sA+9iBPUUwSMMdmKumA78vVZc2GKQKaPUoyAIFYtZAiWEDOkCLhMEa1qCEwbMV0N9fcRimCuofO9iQgmMybypiO7jXK+sH7ulD5KEKsf+nYvIaKqGPAVgb/rH89UVwTlC34jriEA6Esa3jhKy0VnoO30guoIZPooKQKCWK2QIVhCLNtzDUV1BZbjysygyWyx4ti86VZUEItgcb1eQwDQl/IMQdFywoZgAe4dg2IEBLHqoW/3EmL5iiAe0WA5XMYIJqoogkI9RTCXIUhGMJ4xkbcc2XICWJh7RydFQBCrHiooW0JEsDimq5jKmcj5imCiRrC4XBFs6onj/dduxtt39Ne9Tl/SQKZow3bLXUNNZA0toBiNIIhzAzIES4hIH00YKkbTrkzvrB4srjQEqsLwx7dfNOd1+lMGAKBQHiNoprKYFAFBrFpom7eECEMQ811DOekaqhIjsBxE58gOqkV/0pA/B2cUN9NriBp+EcTqhb7dS4ioLI77O30xW3g6b8Fxw03oCpaDWJ25A/XoCxiCeESTQeKFdR8VryVFQBCrFTIES4joLyT6B83kLQBejv9ULuweypkO4pGFee76UqUujVFdKdUCLKSyWPXulWIEBLF6oW/3EmI64bYRswVLPleeOZS3nDmzg2rRmygpgpiuNuXnDw6vJwhidUKGYAkRWUNCEaQLNrring9/oqyWoFAlWNwoEU2R541F1KY6iAoDQJXFBLF6oW/3EmI5pToCAJjNW9jQ7dUGlGcOeZXFC//nEXECQ1Obcw1RQRlBrHro272EWG5YEdguxwZ/8ljQNWQ5LmyXL1gRAF5RGeArgiYWc0NToasMiQVmMBEEsfKhOoIlxLLDwWIAWNsVBWPhorLy6WQLQSiCmB50DS1MEXztg9dg52BqwfdCEMTKpilFwBj7FGPsZcbYi4yx7zHG1vmPM8bY5xhjR/znrwi85v2MscP+f+9v9g2cS4imc7FANlDK0NAV00P9hkSh2UKzhoBSUVlUV2S/IH2BmT97t/agM67PfSBBEOckzbqG7uOcX8I5vwzAtwH8d//x2wBs9/+7G8AXAIAx1gPgDwFcDWAvgD9kjHU3eQ/LSsFyMJ2rrAyuhukHi4NulmhERW/SCLmGRKFZK2IEzWYNEQSx+mnKEHDOZwO/JgCIqqjbATzAPZ4B0MUYWwvgFgCPcc4nOedTAB4DcGsz97DcfOaxQ/ilL+1r6FjbqZw6FtNVDKQMjMyWxlUK11AzMYItvQmoCkNXPEIzBQiCqEvTKwNj7F7G2CkAv4ySIlgP4FTgsCH/sVqPVzvv3Yyx/Yyx/WNjY83e5qIxli5ivEqLiGqUZw15P6tY3xXD0FRePiYUQTMxgtsuWoPHPvo29KeMUoyApowRBFGFOQ0BY+xxxtgrVf67HQA455/gnG8E8DUAHxEvq3IqXufxygc5v59zvodzvqe/v363zeXEcnlFe4iax5bVEQDeYr++O4bRdBFF2zMAWb/1RNJYeIxAURi29ScBUAooQRD1mXOl4Zzf1OC5/gHAd+DFAIYAbAw8twHAGf/xG8oef7LB869IbMcNDaWvh1XDNbSh20shHZ4uYEtfAjnTMwSJJgxBEEPzrkfVwQRBVKPZrKHtgV/fA+B1/+eHAdzlZw9dA2CGcz4M4FEANzPGuv0g8c3+Y+cs9jwUge1wqAoLDaSPRzQ5eez0tOceyhQ9ZZBoImsoCM0UIAiiHs2uNJ9mjO0E4AI4AeB/9x9/BMC7ARwBkAPw6wDAOZ9kjH0KwHP+cZ/knE82eQ/Liu0XfzWC5bjQFBYK2sYiCgZSUQDAaT9OUFIErSnioqwhgiDq0ZQh4Jz/xxqPcwAfrvHcVwB8pZnrriTmowgshyOiKiFDENVVrOmMQmHA0FQOQKk9datcQzSAniCIepCvoEksx4Xjcni2b+5jNZWFFuSYrkJXFQx2RDHku4ayRbvChdQMBrmGCIKoA60MTSI6ijaiCmzXhV6mCEQq6YbumHQNZYsO4hEVjLVmB0+tpAmCqAcZgiYRjeQaiROYNq8wBKJobH1XTAaLs0W7qdTRckp1BPTPTRBEJbQyNImoFm5cEYRdQ1G/jcT67hjOzhRgO64/nax13T4pWEwQRD3IEDSJcA01ogi8GEFJEagKk7v19V1x2C7HSLqITKsVAbWYIAiiDrQyNInlNq4ILCfsGorppTjAen9AzZnpPHKm3VTn0XJKdQSkCAiCqIQMQZOUFMHc1cWW47mGVIVBYeFeQr0Jb5DMZNZEpui0LHUUQFOjKgmCWP3QytAkQgk0FCPwFQHguWmCbaZ7fEMwlTWRM+2WFZMBlDVEEER9yBA0iegfJJTBXMcK90xEVRDXS7v+7rivCHImskW7pYpgsMMrWBPGhiAIIggZgiax56EILMcNZfBEg83nIipiuoqprIls0WnpjOCrt/bg6Y+/Uza3IwiCCEKGoEmkImgofZRLRaCrCmJ6+OPvjuuYyJjIW62NETDGsKYz2rLzEQSxuiBD0CTzqSw2bTcUIyjPDOpOROSAmlZ1HiUIgpgLMgRNIrKFGskast1SsNjQldBcAsDz4YvGc61UBARBEPWg1aYJOOdy6lijMQKRufPff363HLKS9m4AABSOSURBVDAv6I5HcNafXdzKrCGCIIh6kCFoguDi31CMwOEyl/+GnQMVz/ckIhCnaWVBGUEQRD3INdQEwcW/EUVQDMQIqtEV1+XPpAgIglgqyBA0QXBW8Vx1BJxzzBYsdERr7/SDef4ULCYIYqkgQ9AEzjwUQd5yYNouuuK1i7q6A89RsJggiKWCDEETWE4wRlA/a2g6ZwHwagVqEVIE5BoiCGKJIEPQBMHFfy5FMJUzAaCuIgjHCEgREASxNJAhaALbaTxraL6KIK6TIiAIYmkgQ9AEwWBxo4qgu07jNxEjiOoKtYwmCGLJIP9DE9jzqCOY8hVBVx1FENVVxP3mcwRBEEsFGYImCCuCOYLFWT9GEKvfCro7HoFKk8QIglhCyBA0QShGMEcdwVTOQiKiyjbUtehO6A3NNiAIgmgVZAiaYD5ZQ9M5s27GkGDXmg7kLafpeyMIgmgUMgRNYM0ja2gqZ6I7UTs+ILjvjkuavi+CIIj5QIagCeZTWTydt0KVw7VgjOIDBEEsLW2Ro3h8PIvvvnK25ecN9RpqoI6gEdcQQRDEUtMWhuCBn57Axx56seXnDQZ158oamsqZdYvJCIIglou2MASZooWiPfcEsfkSDBbXUwSOyzGTJ0VAEMTKpC0MQdZ0YLscbgMzA+ZDMFjs1En5nM1b4Lx+ewmCIIjloi0MQd700jFNp7WqoFFFUGo4R4aAIIiVR1sYgmzRBhAO7raCkCIIGIKzMwWYAVdUqb0EuYYIglh5tIUhEAVaVosrdqt1H82ZNt7550/iwedOyuemRcM5MgQEQaxA2sIQCEVgtjhgHK4s9n4+MppB1nQwMluQz0010IKaIAhiuWgLQ5AzhSJYPNeQUASHRzIAgLwZcA1l525BTRAEsVy0lSFodbBYqICIqsgYwaHRNAAgb9nyuDMzeSQiKlI0dYwgiBVImxiCxXENCUVg6EoVRVBqHDc8XcDarhi1jyAIYkXSEkPAGPuvjDHOGOvzf2eMsc8xxo4wxl5mjF0ROPb9jLHD/n/vb8X162HarlywW+0aEsHiqK7KOoJDI54iyAUNwUweazujLb02QRBEq2jaV8EY2wjgXQBOBh6+DcB2/7+rAXwBwNWMsR4AfwhgDwAO4HnG2MOc86lm76MWwZ15yw2B64IxzzVkuxzZoo2hqbx33UAr6TMzBexa09HSaxMEQbSKViiCzwD4PXgLu+B2AA9wj2cAdDHG1gK4BcBjnPNJf/F/DMCtLbiHmmTNkq++1W0mLIdDVxRoKoPjunhzLCOfk0VstovxTBFru0gREASxMmnKEDDG3gPgNOf8pbKn1gM4Ffh9yH+s1uOLRi6kCFpdR+BCUxlUhcF2OQ758YFt/QmpCEZmC+AcWNcZa+m1CYIgWsWcriHG2OMA1lR56hMA/gDAzdVeVuUxXufxate9G8DdALBp06a5brMmuYAisFpeR8ChKQyawuC4HG+OZaCrDLvWpPD6sBcrODPtuYpIERAEsVKZ0xBwzm+q9jhj7GIAWwG85GfDbADwAmNsL7yd/sbA4RsAnPEfv6Hs8SdrXPd+APcDwJ49exa8lQ8qglanj1qOC11VoCpejCBTsJGK6kgamlQEwzNeYdlaUgQEQaxQFuwa4pwf4JwPcM63cM63wFvkr+CcnwXwMIC7/OyhawDMcM6HATwK4GbGWDdjrBuemni0+bdRm5AiWISsIU0tKYKi7cDQFMQjmjRAZ2Y8RbCOFAFBECuUxapwegTAuwEcAZAD8OsAwDmfZIx9CsBz/nGf5JxPLtI9AACyxYAiaHWw2HWhKYqMERRtF4amIKqrMlg8PF1AZ0xHPELFZARBrExatjr5qkD8zAF8uMZxXwHwlVZddy7yi+gactygInBRsBwYmop4RIXpuLAdl2oICIJY8az6yuLsYgaLHS9YrCoMtuMpgqiuIKarALxagjPTBTIEBEGsaFa9IWh1+uhLp6bhCZ5SsNirI+AoWi4MTUUsUjIEwzN5rO2iQDFBECuXNjAEJUXQrGvo0Egat3/+J3j6zQkAfvqoyqAqCiwRLA4ogkzBxlTOQn/SaOq6BEEQi8mqNwTZoiO7fjYbLB5PFwEAk35bacvxgsWlGIErYwQAcNafSUBzCAiCWMmsekOQNx0kDA2awppOH836biZRI2A7HLoajBF4iiDqG4Lhac8Q0IhKgiBWMqveEGRNG3FDRURTmlYEYtJZQRgCN6gISumjcT2sCDpJERAEsYJZ9YYgbzqIR1ToqtICRRA2BJbDZa8hx+XSNSSCxaK9RFeMDAFBECuXVW8IsqaNeESDriowA1lD7/mrH+Orz5yY37l8RSDGUHqKwKsjsP1gcVRXZIxAtJcg1xBBECuZVW8Icr4iMAKuIdtx8fLQDJ54bWRe5xJVysEYgeb3Giq5hlRE9TJDQIqAIIgVTFsYgkREg66WgsXpgrezP3B6VtYENEJljMALFotAtCliBH47iWG/z1AHGQKCIFYwq98QFO2KGIEwBOOZIkb9lNBGEFlD0hD46aOqymThWlQvpY9O5yx0RDWoCs0qJghi5bL6DYHluYaCWUOzBUs+f2BopuZr//XF03jsYMl9JGMEZcFiTWEykGxoCgxNgZhTT/EBgiBWOqu+JWau6CBuiGBxFUNwegYvnprG3q09eNuO/tBrv/jDo+hJ6HjX7kHvXGZl+qjudx8VHiZDV8AYQ0xXkTMddFHqKEEQK5xVbQgsx4XpuIjrKiJVXEOawvD3z5zAZNbEHbMbKgxBtmgjopVEU0YqAhF0LikCgaF5biFhCDopPkAQxApnVbuGhN8+bmgh15AwBJdu7JLtIoTbJ/x6G7nA4+J8BVO4hkoTygRR3ftZ1BKQa4ggiJXOqjYEnHPcdMEAtvUl/Kwhz38zm/dcQzfuGkBEVdCXNORuP0imaIcMhDimYJeyhkQdgSCoCABKHSUIYuWzqg1BVzyCL7//Krxj10DVrKEPXb8NP73nRuxck6xQBLbjNZELGoicqCMwy+sIgobA+0hF5hA1nCMIYqWzqg1BkLBryJKZRL1JA4mIFhppCXjZRoDnDhK1BhVZQ4HKYoEoJhP/7yTXEEEQK5z2MQRlWUMd0dJOPRnVKlxDYtEXs4g554FeQy4cl4NzeL2G1NqKgFxDBEGsdFZ11lCQiBZ2DaWipbeeNLTQSEsgHDzOFm1wrsH1U0QLlgPb9c6lq0o4RlARLCZDQBDEyqZtDIGuhrOGgoYgYWgVMYJMwFWULToQjShSUQ0504HtB569mcUlYVUKFnvnJ0NAEMRKp21cQ16w2M8aKlih/j9JQ4PlD5YRBNNGs2Ype6gvacBxuYwTaGWKoJQ+6v2/M0YxAoIgVjZtYwgiWilG4CmCkiFIREozhgWZMteQCCb3JSPyHADkhDKBUASi8RwpAoIgVjrtYwhUBtMP+qYLVoVrCEAocygYM8gUbfl7b8IbRJ+R1ckK9CrB4q64joimUGUxQRArnraKEQBeFtBs3g5nDfmGIKwCAm4is/Rzr1QEXlGapjIorNIQ/Mo1m/HW8/vkdQmCIFYqbWMIRM+gTMGG6bjVFYEZdgcJMkVbNpXrTXqKQDSuiwQWek1h0PzfO6I6LtnQtQjvhCAIorW0jSEQO/OJrDd/oKOKISiPC1T7WcQITk+LMZS6jBcYGu3+CYI492iblUv3F+nxjNdkrjxrCChb/E1H7vazgZ5DIkYwNJUD4GURiawhw68mJgiCOJdoG0Ng+Iu66DYaKiiLVjEERRtdcR26ypA1HRknEIpgaMobQ9mfMmTWUJQUAUEQ5yDt4xrSvMV6IuO5hoLpo8mIcA2VgsKZoo2koaFou9JAaAqTSuLUpKcIehIRaCopAoIgzl3axxCoZa6hYB2B4S3g5Yog4RuCTNEGgxdLEO2lT0/lfcVQmkdAMQKCIM5F2mblitRxDWmqN2e4PEYQj6hIGCpyRQeZooNERJU9hNJFG70Jz01EMQKCIM5l2kcRaGVZQ2WFXklDq8gaWtMR9VxDpg3GPEUQ1UqLfZ+fSipiBKQICII4F2kbQyAUwenpAiKaIttKCMobz5W7hsQx0UhpsReGQCNDQBDEOUzbrFyioOyNs7PY1pcAC1QDA94inylzDSUMzzWULdo4PpHFhu4YIqoC0VpIZBCVFAG5hgiCOPdoG0MggsUFy8V5A8mK55OGWuEaSkQ0JCIaxjMmhqby2D6QAmNMTh/rlYrAO7foPEoQBHEu0TYrV7Ax3Hn9lYbAcw156aOuy5EzHSQMDQlDw2TWBOfAjkHvdSJzqDJGQIqAIIhzj7YxBEH//flVFUEpRiB6DnmuoVIYZftgCkBpHrFwDZXqCNrm4yQIYhXRNsHiYBfQ8/oTFc8Hs4ZEFXHC0ORUs4iqYEtvHEDJBdRbpgiipAgIgjgHaWoLyxj7I8bYacbYi/5/7w48dw9j7Ahj7A3G2C2Bx2/1HzvCGPt4M9efDyJYzBiwra+Wa8gzBMIgJA1NDpjZ1p+QnUVFLUF/edYQKQKCIM5BWqEIPsM5/7PgA4yx3QDuBHAhgHUAHmeM7fCf/jyAdwEYAvAcY+xhzvnBFtxHXYQiWN8Vkwt5kIShIWs6cF0uDUI8oqFoeIog6E6KyWBxedYQGQKCIM49Fss1dDuABznnRQDHGGNHAOz1nzvCOT8KAIyxB/1jl8wQVIsPAF7WEADkLEcGjROGCtP2PqIdfnwA8GIEMb0UP9BkiwlyDREEce7RCkPwEcbYXQD2A/gY53wKwHoAzwSOGfIfA4BTZY9f3YJ7mBOxWz+/SsYQUJpJ8J7/78co+nGBRESTP4uMIcAzBEINAKQICII4t5nTEDDGHgewpspTnwDwBQCfAsD9//85gA8AYFWO56gek+A1rns3gLsBYNOmTXPd5pxEdRX/7ZaduOXCwarPv31HP9572To54P5tO/qwc00Kjsvxoeu34vrt/fLY91+7BWOZgvy9LxnB7960HTfXODdBEMRKhnFedR2e/4kY2wLg25zzixhj9wAA5/xP/OceBfBH/qF/xDm/xX88dFwt9uzZw/fv39+S+yQIgmgXGGPPc873zHVcs1lDawO/vg/AK/7PDwO4kzFmMMa2AtgO4FkAzwHYzhjbyhiLwAsoP9zMPRAEQRDN0WyM4E8ZY5fBc+8cB/CbAMA5f5Ux9hC8ILAN4MOccwcAGGMfAfAoABXAVzjnrzZ5DwRBEEQTtMw1tJiQa4ggCGL+LIlriCAIgjj3IUNAEATR5pAhIAiCaHPIEBAEQbQ5ZAgIgiDanHMia4gxNgbgRBOn6AMw3qLbaSV0X/Njpd4XsHLvje5rfqzU+wIWdm+bOef9cx10ThiCZmGM7W8khWqpofuaHyv1voCVe290X/Njpd4XsLj3Rq4hgiCINocMAUEQRJvTLobg/uW+gRrQfc2PlXpfwMq9N7qv+bFS7wtYxHtrixgBQRAEUZt2UQQEQRBEDVa1IWCM3coYe4MxdoQx9vFlvI+NjLEfMMZeY4y9yhj7Hf/xP2KMnWaMvej/9+5lur/jjLED/j3s9x/rYYw9xhg77P+/e4nvaWfgc3mRMTbLGPvd5fjMGGNfYYyNMsZeCTxW9fNhHp/z/+ZeZoxdscT3dR9j7HX/2t9kjHX5j29hjOUDn9sXF+u+6txbzX87xtg9/mf2BmPsliW+r68H7uk4Y+xF//El+8zqrBFL83fG///2zi7EqioMw8+HlpBZkWVI9jMjduFVDiJC6U0RKeX0A2EEDtRNoBcSQcKA9wp1FwVRZGEZUdHcBEIXdaWFU5MTlo4WNHgcwaCCoN+3i/Ud2h7PPhPGrLU553tgc9b+Zh/2y7vWXt9ea+8zS+rLjfRvrk8Dw8CVwBSwtpCWlcCIl5cBJ4G1pMV6nm2AV98DN3TE9gN7vLwH2Fe4Ls8Bt5XwDNgMjADT8/kDbAU+Iq3StxE4mlnXfcBiL++r6Lq9elwhz7rWnV8LU8ASYMiv20W5dHX8/Xlgb27PevQRWdpZP48INgAzks5I+h04BIyWECKpJWnSy78AJ/h3DeemMgoc8PIB4KGCWu4BTkv6Pz8qvGwkfQr82BGu82cUeEOJI8B1HQs4LaguSYcl/em7R4BVC3Hu+ajxrI5R4JCk3yR9B8yQrt+suszMgMeAtxfi3L3o0UdkaWf9nAhuBn6o7M/SgM7X0pKe64CjHtrlQ7vXck+/VBBw2MyOWVorGuAmSS1IjRRYUUgbpJXsqhdnEzyr86dJ7e5J0l1jmyEz+8LMPjGzTYU0dau7pni2CZiTdKoSy+5ZRx+RpZ31cyKwLrGir0iZ2dXAe8BuST8DLwGrgTuBFmlYWoK7JI0AW4CdZra5kI5LsLSk6TbgXQ81xbM6GtHuzGyctDrgQQ+1gFslrQOeAd4ys2syy6qru0Z4BjzOxTcc2T3r0kfUHtoldtme9XMimAVuqeyvAs4W0oKZXUGq4IOS3geQNCfpL0l/A6+wQMPh+ZB01j/PAx+4jrn2UNM/z5fQRkpOk5LmXGMjPKPen+LtzszGgAeAJ+QTyj7tcsHLx0jz8Hfk1NWj7prg2WLgEeCddiy3Z936CDK1s35OBJ8Da8xsyO8qtwMTJYT43OOrwAlJL1Ti1Tm9h4Hpzu9m0LbUzJa1y6SHjdMkr8b8sDHgw9zanIvu0prgmVPnzwSww9/q2Aj81B7a58DM7geeA7ZJ+rUSv9HMFnl5GFgDnMmly89bV3cTwHYzW2JmQ67ts5zagHuBbyTNtgM5PavrI8jVznI8ES+1kZ6snyRl8vGCOu4mDdu+Ar70bSvwJnDc4xPAygLahklvbEwBX7d9ApYDHwOn/PP6AtquAi4A11Zi2T0jJaIW8AfpTuypOn9IQ/YXvc0dB9Zn1jVDmjtut7OX/dhHvX6ngEngwQKe1dYdMO6efQtsyanL468DT3ccm82zHn1ElnYWvywOgiAYcPp5aigIgiD4D0QiCIIgGHAiEQRBEAw4kQiCIAgGnEgEQRAEA04kgiAIggEnEkEQBMGAE4kgCIJgwPkH2SqpewyoIzUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "actor_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.001}\n",
    "critic_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.002}\n",
    "agent = PPOAgent(env, actor_kwargs=actor_kwargs,\n",
    "        critic_kwargs=critic_kwargs, batches=50)\n",
    "\n",
    "# 训练\n",
    "episodes = 200\n",
    "episode_rewards = []\n",
    "for episode in range(episodes):\n",
    "    episode_reward = play_montecarlo(env, agent, train=True)\n",
    "    episode_rewards.append(episode_reward)\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "# 测试\n",
    "episode_rewards = [play_montecarlo(env, agent) for _ in range(100)]\n",
    "print('平均回合奖励 = {} / {} = {}'.format(sum(episode_rewards),\n",
    "        len(episode_rewards), np.mean(episode_rewards)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 异策算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OffPACAgent:\n",
    "    def __init__(self, env, actor_kwargs, critic_kwargs, gamma=0.99):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = gamma\n",
    "        self.discount = 1.\n",
    "        self.critic_learning_rate = critic_kwargs['learning_rate']\n",
    "        \n",
    "        self.actor_net = self.build_network(output_size=self.action_n,\n",
    "                output_activation=tf.nn.softmax, **actor_kwargs)\n",
    "        self.critic_net = self.build_network(output_size=self.action_n,\n",
    "                **critic_kwargs)\n",
    "        \n",
    "    def build_network(self, hidden_sizes, output_size,\n",
    "                activation=tf.nn.relu, output_activation=None,\n",
    "                loss=tf.losses.mse, learning_rate=0.01):\n",
    "        model = keras.Sequential()\n",
    "        for idx, hidden_size in enumerate(hidden_sizes):\n",
    "            model.add(keras.layers.Dense(units=hidden_size,\n",
    "                    activation=activation))\n",
    "        model.add(keras.layers.Dense(units=output_size,\n",
    "                activation=output_activation))\n",
    "        optimizer = tf.optimizers.SGD(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "      \n",
    "    def decide(self, observation):\n",
    "        probs = self.actor_net.predict(observation[np.newaxis])[0]\n",
    "        action = np.random.choice(self.action_n, p=probs)\n",
    "        return action\n",
    "        \n",
    "    def learn(self, observation, action, behavior, reward,\n",
    "            next_observation, done):\n",
    "        observations = np.float32(observation[np.newaxis])\n",
    "        pi = self.actor_net(observations)[0, action] # 用于训练评论者\n",
    "        q = self.critic_net(observations)[0, action] # 用于训练执行者\n",
    "        \n",
    "        # 训练执行者\n",
    "        x_tensor = tf.convert_to_tensor(observations, dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            pi_tensor = self.actor_net(x_tensor)\n",
    "            loss_tensor = -self.discount * q / behavior * pi_tensor[0, \\\n",
    "                    action]\n",
    "        grad_tensors = tape.gradient(loss_tensor, self.actor_net.variables)\n",
    "        self.actor_net.optimizer.apply_gradients(zip(grad_tensors,\n",
    "                self.actor_net.variables))\n",
    "        \n",
    "        # 训练评论者\n",
    "        next_q = self.critic_net.predict(next_observation[np.newaxis])[0,\n",
    "                action]\n",
    "        u = reward + self.gamma * (1. - done) * next_q\n",
    "        u_tensor = tf.convert_to_tensor(u, dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            q_tensor = self.critic_net(x_tensor)\n",
    "            mse_tensor = tf.losses.MSE(u_tensor, q_tensor)\n",
    "            loss_tensor = pi / behavior * mse_tensor\n",
    "        grad_tensors = tape.gradient(loss_tensor, self.critic_net.variables)\n",
    "        self.critic_net.optimizer.apply_gradients(zip(grad_tensors,\n",
    "                self.critic_net.variables))\n",
    "        \n",
    "        if done:\n",
    "            self.discount = 1.\n",
    "        else:\n",
    "            self.discount *= self.gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RandomAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        \n",
    "    def decide(self, observation):\n",
    "        action = np.random.choice(self.action_n)\n",
    "        behavior = 1. / self.action_n\n",
    "        return action, behavior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均回合奖励 = -16620.0 / 100 = -166.2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD8CAYAAACVZ8iyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsvXmcZOdV3/09tVd3V/Usvc++S6PRYmmsxRuO42XMYoFtgsDBNgYUg5UQIC+x4wQDDp83hACJgdgRYHhNpCh+4xgLW7YlEfAqWRrL0iwazWg0i2af3rfaq578ce9Tdbvq1l7VXV39fD+f+kzPrVt1n67ufs4953cWUUphMBgMhrWNZ6UXYDAYDIaVxxgDg8FgMBhjYDAYDAZjDAwGg8GAMQYGg8FgwBgDg8FgMGCMgcFgMBgwxsBgMBgMGGNgMBgMBsC30guolYGBAbV9+/aVXobBYDCsGr7//e9PKKUGazl31RiD7du3c/jw4ZVehsFgMKwaROR8reeaMJHBYDAYjDEwGAwGgzEGBoPBYMAYA4PBYDDQpDEQkZ8UkeMikhORg0XPfUxETovISRF5h+P4IfvYaRH5aDPXNxgMBkNraNYzOAa8G/im86CI7AfuA24CDgH/VUS8IuIF/hR4J7Af+Gn7XIPBYDCsIE2lliqlTgCISPFT9wKPKKWSwFkROQ3caT93Wil1xn7dI/a5LzazDoPBYDA0R7s0g03ABcf/L9rHyh13RUTuF5HDInJ4fHy8LQs1GAyGTuXJF6/xmW+8wnKMJ65qDETkSRE55vK4t9LLXI6pCsddUUo9qJQ6qJQ6ODhYUxGdwWAwdA1fPnKZv37qvFv0peVUDRMppd7awPteBLY4/r8ZuGx/Xe64wWAwGBycnVhkx0DvslyrXWGiR4H7RCQoIjuAPcAzwLPAHhHZISIBLJH50TatwWAwGFYtSqllNQZNCcgi8hPAHwODwFdE5Hml1DuUUsdF5PNYwnAG+IhSKmu/5gHg64AX+KxS6nhT34HBYDB0IVOLKeYSGbavBmOglPoi8MUyz/0u8Lsuxx8DHmvmugaDwdDtnJtcBGDnKg8TGQwGg6EJzk7EAJbNMzDGwGAwGDqQsxMLeD3C5vXhZbmeMQYGg8HQgZybiLF1Qw9+7/Js08YYGAwGQwdyZmKR7Rt7lu16xhgYDIaO4G9+cIlT1+ZXehkdgVKKcxOL7BjoW7ZrGmNgWJMopfjL75xlejG10ksxYP08fuMLR/iDx0+u9FI6gmtzSeLpLDsGjGdgMLSVSzNxfvtvX+QrR6+s9FIMwFwiQyqT4zunJ0llciu9nBXn7ISVVmo8A4OhzcRTWQBm4+kVXokBYGIhCcBCMsPh81MrvJqVRxuD7cYzMBjaSyJt3X0aY1AfyUy2LR00JxcK4bp/OLm6OxR/5cgVHjt6panP6dzkIgGfh7H+5UkrBWMMDGuURMb2DGLGGNTK+HyS1/77J/n/v3+x5e+tPYOhSJB/OHm9pteks50XTsrlFB/9whF++aHn+NBfPculmXhD73Nm3Mok8nja361UY4yBYU2SSJswUb08/L1XmUtk+GobdJZJ2xj8xO2bOHVtoeImms0pPv7Fo9z+ySe4Opto+Vqa4fxUjPlkhjfvG+TpM1O8/Q+/weeeOkcuV5+XcG5y+RrUaYwxMKxJdJhoLmGMQS2kMjn++/fOA/D0mSmStmfVKsYXUojAu1+zGaCsd5BIZ/nlh77PQ997lflEhq8e66wEgKOXZgH4V2/fx+O/+iZu37ae3/zScR565tWa3yObU7w6GVu2NhQaYwwMaxLjGdTHY0evMD6f5AP3bCOeznL43HRL339iIcmGngB7h/vYvD7M379UqhvMxtO8/y+e4fEXr/GJH9vPvuEIXz12taHrfe6pc7x0da7JVZdy7NIsAa+HvcMRtmzo4XMfupMNvQFOXKn9Wpdn4qSyuWVrUKcxxsCwJjHGoD7+8rvn2DnYy/9z6Ab8XuGbL7dW5J1cSDLQF0REePO+Qb77ysQS72NyIck/+cxT/ODCNJ+67zX83Ot3cOjACM+em2J8PlnXteKpLL/5peP85bfPtfR7ADh6cZYbRiMEfNbWKiKMREN1hbPymUQbjTEwGNpOImOyiWrluVeneeHCDB983Xb6gj7u2Laeb56aaOk1JhZSbOwLAPCP9g0RS2V59qzlfSTSWX7hc4c5P7XIX/3cnfzYrWMAHDowglLwxIvX6rrWhWmrG+ixy7Mt/A6swrljl2c5sKl/yfHR/hBXGjAGOwZXkTEQkZ8UkeMikhORg47j20UkLiLP24/POJ67Q0SOishpEfmULMdwT4OhiKTtGcwnMmTrFPfWGn/1nXNEgj7ec7sVz3/T3kFOXJnj+nzrxFvtGQDcs2sjAZ+Hvz95nVxO8Wuff57nL8zwn3/qNbx+90D+NTeMRNi+sadu3eD8pGUMTl2bb2mB2/nJGPOJDDcXGYOR/hBXZ2vPKjo7sUhvwMug/XksF816BseAdwPfdHnuFaXUbfbjw47jnwbuxxqFuQc41OQaDIa60WEigHkjIpfl2lyCx45e4Z+8dgu9QWsW1pv2DALwrRZ6B07PoCfg464dG/iHk9f5j18/yWNHr/Jv3nkjhw6MLHmNiHDowChPvTJZV4rweXtoTDqrauqFdG5ikd/72ktVRXMtHhcbg9H+ENOx9JLfuUqcnVhkx2Avy32f3JQxUEqdUErV3ExEREaBqFLqKWVVZHwO+PFm1mAwNILOJgKYi2dWcCWdzX9/+jxZpXj/Pdvyx/aPRhnoC7RMN0iksywkM3nPAKxQ0Svji3zmG6/wvru28gtv3OH62kMHRsjkFE+cqD1U9OpUDJ2+f7yGUNEXnrvIp//hFf7w8VMVzzt2aRa/V9g7HFlyfMQuHKtVNzg3ubjsegG0VzPYISI/EJFviMgb7WObAGfFykX7mMGwrDjv0oxu4I5Sis8fvsBb9g2xzbE5eTzCG/cM8q2XJ+rOn3dDF5w5wyJvuWEIj8AP7R3kt991U9m75Fs39zPWH+JrdYSKzk/GuHE0SiTo49il6lk+xy9b5zz4rTN895Xy3tDRS7PsGymIx5rR/hBATbpBKpPjwlRs2TOJoAZjICJPisgxl8e9FV52BdiqlHoN8GvAwyISBdx+omV/m0TkfhE5LCKHx8dXd4m6obOIG2NQlUszca7NJXnzvsGS5960d4CpxVR+o2yGCbsVhQ4TgTXq8bFfeSP/7WfvwFdhuIuI8I4DI3zz5QkWkrV5eK9Oxdi+sZcbx6I1icjHLs1y6KYRdmzs5dc//4JrSEopxbFLsyUhIrA0A4Crc9V1gwvTMXJq+UZdOqlqDJRSb1VKHXB5fKnCa5JKqUn76+8DrwB7sTyBzY5TNwOXK7zPg0qpg0qpg4ODpb+QBkOjOMNE3WQMUpkcn/zyi/m77WZ44YK1Ud66ZV3Jc2/Ybf09tiJUpKuPB4oE0xtGooT83qqvf+eBUVKZHH//UvU2Ftmc4uJ0jK0bezgw1s+JK3MVEwiuzye4Pp/k4Pb1/Of7bmN8Psm//dKxkr5DF6bizCUyJZlEACPR2j2Dc/kGdR1oDBpBRAZFxGt/vRNLKD6jlLoCzIvI3XYW0fuBskbFYGgXiUyWsL3RdJMxOHFljr/49lm+1mAxlpMjF2cIeD3cMBIteW4wEmT/aJRvnGreGGjD5fQM6uGObesZ6AvU9D1fmY2Tziq2bujhwKYoiXSOM+MLZc/Xns+BTf3csnkd//Kte/jbFy7zpeeX3sOWE48BeoM+oiFfTZpBPq10tWkGIvITInIRuAf4ioh83X7qTcAREXkB+F/Ah5VSui/tLwF/DpzG8hi+2swaDIZGSKazDEetO9FuMgYz9vdy+nr5Da5Wnr8ww41j0ZIYuOZNewd57vx009lYOkxU7BnUitcjHDowwhMnrvH0mcmK575qp5Vu29DDTWPWxl0pVHTc3uT3j1kG8ZfevJvXbl/Pv/ubY0s296O2eLxvJOL6PqP94Zo8g4vTcfqCPtb1+Kue22qazSb6olJqs1IqqJQaVkq9wz7+BaXUTUqpW5VStyul/tbxmsN2mGmXUuoB1Y5+uAZDFRLpHOt7A/g80lX9iWZi1sb68vXmxkdmc4qjl2a5bXPpna7mjXsGyOQU3z/v3pri2KVZ/uDxk1VbOU8sJOkL+moKCZXjV9+6l60bevjQXz3L9yvMQzg/ZRmDrRt72DXYS9Dn4XgFEfnYpTm2bewhGrI2Z69H+P333koqm+PfOcJFxy7Nsnc4QtDn/j1YtQbVjcGlmTib1oWXPa0UTAWyYY2SSGcJ+bz0h/1d5RnM2d/LqWvNeQavjC8QS2Vd9QLNTrtC9uK0uzD6hecu8sf/5zQXpioLp5MLKQYaDBFpNvYFefgX7mIoEuSDn32WIxdnXM87PxnD7xVG+8P4vB5uHK0sIh+/MsuBsaUGcftAL7/+9r088eI1Hjt6FaUsw+kWItLUWoV8aTrOpvXLN8PAiTEGhjVJIpMl5Pd0nTGYsTNdxueTeS+hEZ6/YG2mt2wubwyGIiG8Hil7x3tlxjpeLXQzsZBkYwuqbYeiIR7+xbtZ1+vnZ//iGdcaglenFtmyvgevXWhw01iU45fnXL2X2ViaC1NxbtpUqpl86PU7uHlTP5949BjHLs0xG0+7iseakf4QEwvJqhXP2jNYCYwxMKxJEukcIb+XaNifv5vuBmYc30szusELF2aIBH0V8929HmE4EuRymVYLV+zjtRiDZj0Dzdi6MA//wt30BLz8+udfKHn+/GSMLRsKoyQPbOpnPpFx9V6OX7GMyU1jpZu8z+vh995zCzOxNL/00PcBd/FYo2sNrs2V9w4Wkhlm42njGRiWj3gqW3enx24jkc4S8ndfmGgmlsbvte56X27CGBy5OMstW/qrTtoaXRfOewDFXJ4teAaVdAMrTNS6PjxbNvTwwddt56Wr81x3bL5KWXMCtm10GIMKIrLWEm4aK/UMwBKVP/xDu7g4HcfnKS8eg6MKuYIxuGSH24xnYFg2fuWRH/DuT39npZexolieQfeFiWbjaXYN9hH2e2vqu+NGIp3lxJW5iiEizUh/KO8BOElmrBuOoUiQy7OJsrpBJptjKpZqSZjIyet2WQ3tnnJ4JTOxNPPJDFsdnsHekT58HuHYpVJjcOzyLKP9oYqG6oG37Gb3UB83beqvKIDXUoV8acYSt41nYFgWfvDqNI+/eI0LU/Eld01rjWQ6S9DnJRr2dVWYaDaeYn1PgN1DfQ2HiV68Mkcmp7i1BmMwZgujxXf+12Ytz/PHX2N1mykXKpqOpVEKBlsUJtLsH4vSH/bz3dOF6+pMImdrjaDPy57hiGsl9fHLc64hIichv5fP/7N7+LP331HxvHwVcoXupdoz2Gw8A8Ny8AePn8qLZ61oJbBasQRkK0w0l8hUTX9cLczE0qzr8bNnqI+XG8woOmKLx7dVyCTSjPaHSWZyTBe1aNA6wht2DzDQFyhrDAoFZ631DLwe4e6dG/iOo5eQ7lbqDBMBHBiLcuzS7JLfgVgqwyvjC2VDRE429AYYioQqnhMJ+ugNeCt6Bhdn4gS8npaGzOrBGIM1xFOvTPLt0xM88I92A7i6xmuBbE6Rzqp8mCibUzX3tel0ZuOWMdg93MfVuURDNRQvXJxlOBrM381WYmyddc7logH2OnQ0ti7MXTs38lQZ3WCyyYKzSrxu1wAXp+NcsD0CXXC2Zf1SY3DTWJTJxRTX5go62okrcyhFxQyhehCRqrUGl6bjjK0LVdVp2oUxBmsEpRR/+MRJhqNBfunNu9gx0LtmPQPdsVR7BtAdVchKKWbiafrDAfYOWWJmI97BCxdmatILwPIMoDQWftkWlcfWhbh750auzCZ41d6UnTTbiqISr9u1ESDfafT8VIyhSJBwYGlsX2/4Tu+l0IaiumdQK9WqkC/NrFyNARhjsGb4xqlxnj03zQNv2UPI72V/jR0bu5G8MfB5usoYJNI5Upkc/WE/e4b7ADhdZyXybDzNmYnFmkJE4BRGSz2DdT1+egI+7tm5AXDXDSbKNKlrBbuH+hiMBPnuK9Z1izOJNDdv7mf3UB8f/+LRfDX1sUuzbOgN5JvMtYJaPIOVyiQCYwzWBEop/uDxU2xaF+anDm4BrJS6i9PxuiZEdQt6/rGuM4DuMAYzcSvksq7Hz+b1PQR9nro9g6MX7U6lNXoGA31B/F4pueO9MpPIew27Bvts3aC0TcTEQoqA10M05KtrnbUgIrxu10a++4oVonp1KsbWDaV1E0Gfl4d+4S4GI0E++NlnOHJxhmOX5rhpLNrSthCj/SGuzyfIZEsLz5KZLNfnk2xaV2qslgtjDNYAn3vqPEcvzfIrb92Tbzqm3d9aJj11G84wke450w3TzrRBWxf24/UIuwb76q41eMFu43BzhZ5ETjweYTga4kqRZnBpJs6Y7TWICHft3Ohab2BVHwfa1ovndbs2Mj6f5PjlOa7OJVw9A4Bhu3q5v8eqXn75+nzL9ALNSH+InIJxl/biulbDhIkMbSGRzvKx/32ETzx6nDfsHuDdrykMlaulY2O3UjAGhTBRN6SX6lYU+nvaO9zHy3XWGjx/YYadA73596iFsf5wvsBMc2U2wei6QoilnG4waRuDdqHrDR559lWgNJPIydi6MP/jF63q5XRW1ZRJVA+j+fTS0lDRpZmVLTgDYwy6lgtTMd77me/yP565wC+/eRd/9XOvXTIxakNvgLH+0JoUkfVgm5DfS3+Pe5jo5NV5/uUjP3B16TuVvDGwv6c9wxEuzyZqbjGdzSmePTfFHdvW13Xd0XVLC89iKautwphjYyunG0y0uPq4mC0beti8PsyXfmDNH3AWnJU7/+FfvJufvGMzb9zd2oFaI9Hys5DzNQbGMzC0kovTMX70j7/N+ckYf/b+g/zGoRtcRwfetKl/TaaXJh1hor6AD4+UGoOvHrvC3zx/uaZOk53CbF4zsO60dw9ZIvIr44s1vf745VlmYmnesGegrutqYVTPQ85nEvUXNrZyusHkQrLtefWv3zXAvJ06XM0YAOwY6OX3f/LWvFFtFSMVqpAvzsTxCDWl87YLYwy6kCMXZ5mNp/mLD7yWt+0fLnveTWNRzkwsstglOfa1ksgUjIHHI0RdWlKct3PSF1Or57NxagYAe2xjUGuo6FsvWymYr99dnzEY6w+TziomFy1jpL2EUcfGJiLcs2uAb5waz3fuVEoxsZBqa5gI4HW7rRTTvqCPDb3tvVYl1vf4Cfg8rv2JLk3HGY6G8FeY99xump109vsi8pKIHBGRL4rIOsdzHxOR0yJyUkTe4Th+yD52WkQ+2sz1De7oP7bBSOU7rgNj/SgFL11dW6GiQpjI+vV360+kxw+uJkM5E0vj8wg9dh791g09BHyemkXk75ye4MbRaN136sXppboAbawo/v3eOzYztZjia8et8ZRziQypbI7BNnsG9+y0jMHWDT0rMjRGIyJl5xpcnI6tqF4AzXsGTwAHlFK3AKeAjwGIyH7gPuAm4BDwX0XEa89F/lPgncB+4Kftcw0tJGXHuXX3ynLobIljFSY9dSOFOgNr04yG/CWVuufs1gXzifYbg+cvzPCRh59rWp+YsauP9Ybn83rYOdBbk2cQT2U5fG6aN9h30fWgN30dHro8k0DEytBx8sbdA2zd0MNDT58HrBARtKfgzMlQNMQtm/tbWkDWKCPRkGt/opUuOIPmx14+rpTSfy1PA5vtr+8FHlFKJZVSZ7HmHd9pP04rpc4opVLAI/a5hhaiPYNys2s1w9EgG3sDay691CkgQ6lnMBNL5cXYxWS27ev5mx9c4itHrrhW6NbDbCxdkgW0ZzhSk2fw7LkpUtlc3SEiKPUMrszGGewLlvz+eTzCz9y1le+dneLla/NNzz6uh4d/8W5+594Dbb9ONdw8g2xOcXU2seo9AycfojDcfhNwwfHcRftYueOGFpK27zADVeKPImKLyGvUMygTJtIhIlieMJEW8bVO0SizcRdjMNTHxek4sSrax7dPTxDwerhzx4a6r7uhN2DFwu1Nzkordd/YfvKOzQS8Hh763qsFz6C3/cag2RnLrWKkP8y1uYLYDtbAm0xOdb5nICJPisgxl8e9jnM+DmSAh/Qhl7dSFY6Xu/b9InJYRA6Pj49XW6rBJp0PE1W39QfGopy6Nk8y0/474E7BKSADJdPOnJvyfJuNQTan8um9OjTVKDPxVD6TSLN3WIvIlb2Db788we3b1tETqL8SWMfCda3BZUfBWTEb+4K88+YRvvDcRS5MW5/zQGTlRN3lZrQ/tERsh86oMYAajIFS6q1KqQMujy8BiMgHgB8F3qcK5YUXgS2Ot9kMXK5wvNy1H1RKHVRKHRwcbG3ObzdTa5gIrOKzTE413O54NaLDREHfUs9A//qenVhE64zt9gzOjC8Qtz2VZj2DmVg6n0mk2TtsNayrNOhmYiHJi1fmeOOexv/GRvutKmSlFJcdrSjceN9d25hPZPjrp88jAht61o4xGHEpPOuEGgNoPpvoEPCvgXcppZy/yY8C94lIUER2AHuAZ4BngT0iskNEAlgi86PNrMFQSiprbWq+GlrhalFtLdUbWINtPHmhtT/sJ51V+U353OQim9aFCfg8bTcGR+3PPRL0Ne0ZzMbTJbnx2zb2EvB5KhoD3citEb1AM2Z35JyNp4mns/nW1m68dvt69g73cWEqzvqegGsNTLei9ZWzjp91wTNYub5E0Lxm8CdABHhCRJ4Xkc8AKKWOA58HXgS+BnxEKZW1xeYHgK8DJ4DP2+caWkgqkyPg2OwqsXVDD5GQb01VIuv5x5po2AqN6P5E5yYW2THQSyToa/ucg6OXZgn7vbxhzwDnJho3BplsjvlEpkQz8HqEPUN9nKzg+X375XGiIV/Fge7VGOkPcXUuwcVp97RSJyLC++7aBsBAmzOJOo29wxE2rQvzX//+dD577OJ0nI29gZLW2stNs9lEu5VSW5RSt9mPDzue+12l1C6l1D6l1Fcdxx9TSu21n/vdZq5vcCedzVUVjzUiwv7R6JrKKNLzjzXFbazP2a2Oe5fBGBy7NMv+sSg7B3u5OB3P6z31MmenwBaHiQD2DUc4WaaWRCnFt1+e4HW7BvIT8BphdF2YbE5xxO56OlqlkvYnbt9E2O9dFvG4kwj5vfzbH7mRl67O89d2im0npJWCqUDuStLZXNUaAyc3jEQ4dW1hSYZDN6NHXmqcxmB6McVsPM32jb30Bn1tDRNp8fjmTf1s29hLJqdKJobVykxsaSsKJ3tHIlybS+bPcXJ2YpHLs4m6W1AUowVjPQ+gkmcAVm3H7733Fj785l1NXXc1cujACG/cM8AfPn6K8fkklzqg4AyMMehKdJioVvaNRFlIZvKxy24nkc7mC85gqTHQsdzlCBOdnVgglspyYFM/2+0h7ecaFJG1V+PWbXTfiBaRS0NF3zlttaB4QxN6ARQmnj336jR+r9RUVfyuW8f4ob1rLzFERPitd91EIpPlP3z1JcszMMbA0A5S2VxdPU70ZvHS1fraHa9WKoWJdNx++0AvvUFvW42BFo8PbIqy3W6tfL5BEXkmvrRjqZN9dkbRSRcR+TunJ9m0LlyxtXMtaMH47MQiw9GVm+O7Wtg12MfPv2EnX3juIol0zoSJDO2hfs/A3izWSI+iRDpLsEyY6NxkDI9YQ9OtMFH76i+OXpwj5Pewe9Aaz9gT8HJuokHPILa0SZ2T0f4QkaCPU0XGPpdTfO/sJPfs2th0z57+sD9vYMcqpJUaCvzzt+zOayvGMzC0hXoEZLCqM7dsCK8dzyCTW6IZREKFATfnJhbZtN5KK42E2hsmOnZplhtHo/i8VubXto29jXsGFTQDEWHvSKTEMzh1fZ7pWJq7d9bfj8jtGtoIjFZIKzUU6A36+MSP7Sfg9XDDyMr3TTLGoAtJZ1VdngHAvuEoJ9eIMUims4Qcn4/XI0SCPtszWMzH73sDPhba1Kgul1Mcvzy7JJ1z+8aehmsNZu202HKzhPcORzh1bX7J2Mmn7fqCuxpoQeGGNgKVCs4MSzl0YJRjv/0OtjYZpmsFxhi0kW+cGl+RNg+pTH2aAVgZRWcmFtdEW4riOgMotKQ4O+EwBkEf8XSWbBuyrM5MLLJoi8eabRt7uTAVb+h6M/EUkaCvbAHXvuE+ZmJprs8X5u8+fWaKzevDbKlh4EstaCOwyXgGdVHvjVu76IxVdCGvTsb4wGef4StHriz7tVN1ppaCpRtkc4pXrjdXBbsaKBaQwYp5n51cZD6RYfuAZQwi9l12Owbc6IrvYs8glc01lF46GyutPnayzw5DaO9P6wWtCBFpdHqp8QxWJ8YYtIlxuyNjo3njzWAJyPVVM96QzyjqfhG5uM4ALGOgq7B3DFh3yr1ByxjUEir62P8+yl98+2zNazh6aZagz5OfRgaWZwCN9Shy61jqRDes020pWqkXaHSnUqMZrE6MMWgTeh6t0y1fLiwBuT7PYPtALwGvZ03oBm5hov6wP9/gzxkmgtqa1f2fl67x2NHavUCneKzZbhuhRnQDPdimHBv7ggz0BfM/31brBQA/fPMo/+5H93NjB4ihhvoxxqBN6OEo1+eW3xjUm1oKVrvrXUN9XZ9RpJSywkRFn4/uT+QR2Lze2pQj2jOowRjMxTMlAq3zml8+cplnzk6RyuRs8XiupBfQcCRE0OdpKKNoJpZiXbhyn599I315z6DVegFYBvXn37DD1BisUupvXm6oiWltDOZL5522m3SdRWeaG0YiPGXfMXYrSfvuP+jiGYBlCLQh7a3RGKSzOavjadryBIvHPb54ZY4HHv4BAGG/l5s39bOQzJQYA49H2Laxp6EqZLeOpcXsHY7wyDMXyNp6wVtvHK77OobuxXgGbWLWzvu+VqNncHkmzrPnplpy7XRWNWwMrs4l8gVM3Uhhypm7MdDiMUBv0DqnWpjIOSfZbS6E1iI+ee9N/NRrtzATTxHye7hrZ2mIppFaA6VUVc0ArErkeDrL35241nK9wLD6MZ5Bm9DtAcbnkyilqlZ4PvjNM3z5yGUO/9u3NX3tZANhInC2pZjjri7dKArzj0sylMpbAAAgAElEQVSziQB2OPK9I0Hr2EKVKmTnlLSXr8+XNH07cWWOsN/Lz9y1Ld8ZtNzvxI6BXr55apxcTtUcbomlsqSzyrX62In++f5/T50DcDVGhrWL8QzahNYMUtnckvm65VhIZlrW+qDeCmSNroLsZt0g7xn4SusMoJDRAwXPYCFR+ec353jerRnciStz7BuJLGkRXe7mYNvGHpKZHNfqCC/qG49KAjLAHrtH0XdOT7JlQzivjRgMYIxB25hxGIBaMoqSmVzLCr4aEZABhqNB+sP+7jYGGfcw0Xq7jcOOQacx0HUG1TwDK0wU8Ho4fX3pZ6eU4sXLc+wfqy3DJt+9tI4eRboVRbUwUV/Qlx+tePeO7vT8DI3T7NjL3xeRl0TkiIh8UUTW2ce3i0jcnn6Wn4BmP3eHiBwVkdMi8ilptkNWhzITS9FrTy6qJaMomc6SU+SnHzVDvfMMNCLCvpHyg1C6gXJhont2beSTP36ANzpaOQd9HnweqSogz9uewYFNUU5dW1iSUXR5NsFcIsONo7UZA909tJ700kL76upTw3QHU6MXGIpp1jN4AjiglLoFOAV8zPHcK24T0IBPA/djzUXeAxxqcg0dyUwszW77D6+WjKKEneWis10aJZdTZHKNCchQGHTjliLZDZQTkP1eDz9797Ylef8iQl+oen8iHSa6Y9t6ZuPpfMEhwAlbPN4/GqlpfaP9YQJeT33GIFZbmAgKuoHRCwzFNDv28nF7rjHA08DmSueLyCgQVUo9pazd5nPAjzezhk5lJpZir11dWktGUdLepFJNGoOU7Vk02u9k30iEhWQmP8u22ygYg9o+n95A9WlnOkx0xzZrgz3t0A1OXLGMwb4aC7G8HmHLhjDn6wkT1agZAHzwddv51E+/xugFhhJaqRl8CPiq4/87ROQHIvINEXmjfWwTcNFxzkX7WFeRzSnmEhnG1oXpDXhr8gySLfIM9AzdRgRkKLSl6NZKZB0mCtbYrqOvhmlnc4k0HoHbtqwD4OXrBWPw4pU5tm/soS9Ye+Le9o29dXkGOlmhmmYAMBQN8a5bx2p+b8PaoeqOISJPisgxl8e9jnM+DmSAh+xDV4CtSqnXAL8GPCwiUcAtkF02HiEi94vIYRE5PD4+Xs/3taLMOu7UhqOhmgVkaIFnkGnOM9g73N09ipJlBORy9NUw02A+kaEv6GM4GiQa8vGyQ0Q+cWWuZr1As2Ogl7MTizV3L52Npwl4PYRr/J4MBjeq3q4opd5a6XkR+QDwo8A/tkM/KKWSQNL++vsi8gqwF8sTcIaSNgOXK1z7QeBBgIMHD66aILbO7ljfE2AwEmS8jjBRsxlF6az1MTWqGURCfjavD3Oiaz2DOsNEQV++gLAcc/E00bAfEWHPcCSfXrqQzHB+Ksa7b68YPS1h70iEZCbH+clFdg72VT1/Np6iv8ff9LQyw9qm2WyiQ8C/Bt6llIo5jg+KiNf+eieWUHxGKXUFmBeRu+0sovcDX2pmDZ2Icx7tUDS0ImGiRo0BwI2j0Xysu9soZBPVGiaqPgd5LpEmak9L2zvcx2k7THTy6hxKwf46PYNqobrHj1/l6MXZ/P9nYtWrjw2GajSrGfwJEAGeKEohfRNwREReAP4X8GGllO618EvAnwOngVdYqjN0Bc55tEORYI1hIu0ZNGcMkk2GicDavM5NLBKvkl+/GimXTVSOmjSDeCbf6G73UISpxRSTC0levGJt5jfWWGOg2TMUQcS9+C+XU/za51/gZ/786XxNw2w8XbX62GCoRlPtKJRSu8sc/wLwhTLPHQYONHPdTmcmXphHOxQJEktlWUhmKoqIybT2DJoNE2kBufGQwY2jUXIKTl6bz4ui3ULeM6jRWPYGfVUrw+cS6Xz3Tz2f4NS1BU5cmSMa8uWHvtRKOOBlx8ZeV8/gzMQiC8kMIvDBv3yWL/7y65mJpRkzMwQMTWIqkNvA9KLDM4gGAbg2VzlUpCtjV1pAhkJY48XL3RcqSmSy+DxSdjxkMX1BH4upDLkKYu58IpMPE+2xh8icvj6fF48bieXvcxlgD3D00gwAv/eeW5hYSPILnzvM+EKypoIzg6ESxhi0gZl4GhGr381QxLpjq1SFnM2pvPDbCZrB5vVh+oK+rtQN3AbbVKIv6EMpiKXLeweWgGx5fSPREJGgj5euzvPSlfma21AUs28kwrnJ0lDdCxdmCfu9vPs1m/gv972GIxdnGJ9PGs3A0DTGGLSB2ViKaMiP1yMMRSzPoJKI7PQGWlV01owx8HiEG0cjXWoMSucfV6LatLNsTjGfLHgGIsLu4T7+z0vXiaezdaeVam4YiaAUS9JUwRqXeWCTNSHtHTeN8PEfvhGAjX3GMzA0hzEGbcA5gnDIHnQyXkFETjjuOpv1DFoRJgJLN3jp6nzF8MhqJJnO1lxwBuR1nnIisj4eCRX0oD1DfVyZtYx/vZlEGl2x/NKVgjHIZHMcvzzLzZsKOs7Pv2EHD/7sHdz32i0NXcdg0Bhj0AamY4XsjmjIR9DnqZhRlGyhZ6DDTY1WIGtuHI2ykMxwYbr+qVudTCKTrcsz6KviGehZBlFHmGbPkJUa6vMIu4eq1wm4sXVDDyG/Z0lG0cvXF0ikc9yyuTAhTUR4+00jbOwLNnQdg0FjjEEbmI2l6LdbIosIQ9Eg1ysIyM4MomaziVrpGQBdFyqywkS1ewb50ZdlmtXpJnU6TAQFEXnXYF9d13Li9Qh7hyOcvFb4/HVtwc2b+8u9zGBoGGMM2sBMPM16R9OwoUjllhSt9Qya1wzAanXsEfK58t1CIwIylA8T6SZ1WkCGwhCZG2vsVFqOfcORJemlRy7NEAn62OEYwGMwtApjDNrATGxpEdBQJFgxtbSlmkHeGDTXmiAc8LJjoLcLPYNsXT18+kJ6wI27MZh38QzG+kO8ae8ghw6MNrFSK6NoYiHFhN0S++jFWQ5s6q95HKbBUA/GGLQYq2NpOh8mAqpWITsNQKeEicAKFXVbrUH92UR69GW5MJHtGTiMgYjwuQ/dyaEDI02stDCG9OTVeVKZHCeuzC/RCwyGVmKMQYuZi6dRiqWeQTTEfCKzxANwoquPoXVhomYFZID9Y1EuzcRrmuG8WkhksgQbChO5/+wKAnJTxfyu6EE0L12dtwxCNmf0AkPbMMagDr53ZpInX7xW8RzdpG59b8EYDOpagzKFZ0sF5M5ILYWCiPxSF4WKkukcoTpSS8N+Lx6pkE1kh4nqmVdQK4ORIAN9AU5eneOIXXl86+buag9i6ByMMaiDz3zjFf7T4ycrnqPbV69ztAcYtmsNyhWeJdrgGTQrIIOjLUUXGQNLQK79sxEReis0q5uLWz2nam1vUS/WTOp5jlyYZV2PPz/Q3mBoNcYY1EE8nS0b6tE421drClXIy+cZ+FogMg5FgmzoDXSViFxvNhFU7lw6n0gvKThrNfuGo5y8Ns/zF2a4eVO/mVlgaBvGGNRBIp1bchfvRsEzcDEGZTKKtAHoC/pa0I5CEfB5WrJpiAj7R6Oc6KL00kSmPgEZ7GZ1FcJETvG41dwwEiGRznHymhGPDe3FGIM6SKSz+e6i5dDzaNc5sonW9wTweYRrZTwD7W1EQ76WtLBuhXisuXHU6p6ZyTZnpDqBdDZHNqfq0gyAqmGidojHGi0iA9xi9AJDGzHGoA6SmVz1MJHLcHKPRxiMBCsIyNZGGw37WxImaoV4rLlxNEoqk+PMRO0D2juVegfbaCqFidrtGewdtgbdAMYzMLSVpncNEfmkiByxJ509LiJj9nERkU+JyGn7+dsdr/mAiLxsPz7Q7BqWi0Q6SyKdwx717MpsPE005MNbFLO3ag3KhIns0FMk5GtJC+tmC86cdFNbisLIy9aFieYTmbZqBuGAl20behjoCzISNQNsDO2jFbeQv6+UukUpdRvwZeA37ePvxJp9vAe4H/g0gIhsAD4B3AXcCXxCRNa3YB1tJ5GuPppyOpZaEiLSDEZCZTuXJjNZ/F4hHGjeGLTaM9g91IffK12hG+ifXz11BmCHiSr0Joq2eZbAz9y1lQ/cs82Ix4a20vSuoZRy3jL2Avq2+V7gc8riaWCdiIwC7wCeUEpNKaWmgSeAQ82uoxy/+j+f538/d7El7xW3N5NKoaKZWKF9tZPhaPkq5EQ6R9DnJeD1tGSeQSvSSjV+r4ddg32cvLr6PQOtx9QfJvK6homUUtZgmzaGiQDuf9Mu/vk/3tPWaxgMLdk1ROR3ReQC8D4KnsEm4ILjtIv2sXLH28KTL17j2KXmNzKlVD7MUCmjaCaedp06NRQJMbWYct3sk3Zb5aDf03ECMhRy3Vc79c4/1vSFfCymsiXhwcVUlpxqT/WxwbDc1PRXISJPisgxl8e9AEqpjyultgAPAQ/ol7m8lapw3O2694vIYRE5PD4+XstSSwj6vfk7+mZwhm8qeQazsRTrXcJEehaybjpW/N5Bn5dgKzyDFoeJwDIGl2cTzMZWd1uKRgXk3qCPbE6V3AToJnWRNnsGBsNyUNOuoZR6q1LqgMvjS0WnPgy8x/76IuAcv7QZuFzhuNt1H1RKHVRKHRwcHKxlqSWEA56qGUC14OwfVCm9dLpMmGjQHj7iphtYxkB7Bs0Pt2llmAjgRt0wzWVA+2qiICDXn00EpW2s8+2rjTEwdAGtyCZyBjPfBbxkf/0o8H47q+huYFYpdQX4OvB2EVlvC8dvt4+1hbDfWzJUvBGcBqBcmEh3LF3nEibKt0J2iT0n0lkCPk9rNINMe8JEwKrXDQqeQf3ZRFD6s8sPtjFhIkMX0Irf4v8gIvuAHHAe+LB9/DHgh4HTQAz4OQCl1JSIfBJ41j7vd5RSUy1YhyvhFoWJnN5FOU9jPmF1LO13CRP1BnRf/NLXJjPW9K2g39t8C+tsjmigtXeqo/0hIiHfkhGMq5FEgwJyb1nPoHSWgcGwWmnaGCil3lPmuAI+Uua5zwKfbfbatRBqmTGorhnogrP1LmGicMDagGIuQ1KsIe0Fz0Ap1XAaoSUgtzYFUUS4oQtE5IKA3Jow0bydbtrOOgODYbno+grkcMDbEs0gXoNnoJvUuWkGekhKrIxnEPR7Cfo85BRkcuWL2qrRDgEZ7Iyia/MVC+46nfaFiYxnYFj9dL8x8LfGGCwNE7nH9aftJnX94dIwUY/f2lDcjEEinSVkC8jQXOfSdIvrDDT7RqLMJzJcni0/vrPTaaboDMqHiYxnYOgG1oQxWC7NYDZW3jPQYaK4S5goZXsGWvhtRkRuh4AMVvdMWN2Dbpr1DEqMQSJj1YfUGXYyGDqRrjcGQb+XeKr5jpu1aQal7as1AZ8Hv1fKCshWaqnX/n/jxiuVVfjbECbaO1wYwbhaSaRziNQ/ErRcJpg1y8CEiAzdQdcbg1aFiZwbdKLMnXt+sE2ZGHJPwOea5ppwCMjQnGfQjgpksL6nTevCq1pEtsJx3rrF+R7bSBf3J5qLZ4iaEJGhS+h+YxDwEE+XthKol1rCRDMxa+pVuRGIPQGva51BIbW0ec2gXQIyrP62FIlMfSMvNR6P0BvwspBc+nNfjiZ1BsNy0f3GwO8lm1Oks80ZA+cdfTkBeSaWctULND0BLzEXQ5LMtNYzaGULayf7RiK8Mr7QdGHcSpFI5+quMdD0hUrbWC9HkzqDYbnoemOg//ibFZF1aMjvlYqppetcMok0PQEfsaINRRuqoM/btGaQyykyOUXA2x5B84aRCJmc4szEQlvev900Mv9Y4zbtbD6RMZ6BoWvoemOgs3ia1Q306/vDgYphoqqeQZFmoDf+oL/gGTQaJkrZoyn9vvZ5BgAvrdLZBlar8MZ+5d2mnc0l0iat1NA1dL8x8LfKGFjCbG+wvCA9U2awjcbVGDjaKjerGWhj0A4BGWDnQB8+j6zajCKrVXiDYaKiaWfWLIOMCRMZuoY1YwyaDhOlswT9HkI+b3nNIO7epE7TE/CVtKPQG7+uQIalHVLrIW2/V7sE5IBvdQ+6scJEjX02xWGiZCZn9YEyTeoMXULXG4O8ZtBk51J9Vxnye1xbWOupV+XSSqFKmMjnyRsDfYdfL1okb0cFsmY1ZxQ1JSAXGQPTpM7QbawdY9CCMJE1jcw9TJRI58ipQusCN9yMgfYygj5vvpI12eBadZZPu8JEADeM2oNu4qtv0I2uM2iEvqCPuXg6n6I8Z5rUGbqMrjcGrRKQ4ylrI7G6oJbeuS/a4R/dkM6NnqBbmKjQIiHQpGdQEJDbaAzysw1Wn3fQaJ0BwM2b+5lLZPjO6UnANKkzdB/dbwzyYaLmcuMTmSzhgJeQz+N65x6zC5J6AhU8A7+XdFYtydPPawa+5jWDgmfQnmwigAOb+gH4wavTbbtGu2gmTHTvbWMMRoL8t2++ApgwkaH7WDvGoAUCcsjnLdsSO+8ZBCp7BlBcwOZILW1aM2ivgAwwFAmxc6CXZ862bR5R22imziDo8/LB123nWy9P8OLluXyYqN8IyIYuoaldQ0Q+KSJHROR5EXlcRMbs428WkVn7+PMi8puO1xwSkZMiclpEPtrsN1CNUMD6FluhGVTKJtLhn54qmgFALO3ISnEMXMnXGTSaTaTDRG3UDADu3LGBZ85NkW1i7sJKkLR/ho3yT+/aRk/Ay5996wzzCd2+2ngGhu6g2V3j95VStyilbgO+DPym47lvKaVusx+/AyAiXuBPgXcC+4GfFpH9Ta6hItozaFSU1ei7ynLZRIt2mKiiZ2A/t+jocVNILfXg83rweYRUtnMFZLCMwXwis6p0g5SdChqpYKyr0d/j56deu4W/feFy/ns3YSJDt9DUrqGUciac9wLVbhXvBE4rpc4opVLAI8C9zayhGq1LLc3ZxsA9TJT3DCppBoHSMJEztRSsEE/DmsEyCMhgGQOAZ85OtvU6rUQXjFXK9qqFn3/DDhTwyLMX8HulYUHaYOg0mv5NFpHfFZELwPtY6hncIyIviMhXReQm+9gm4ILjnIv2sXLvfb+IHBaRw+Pj4w2tz2/fbTcbJrKyiXRqaa6kC2reM6iQTaS9hkVHRpEztdT619N4NtEyeQab1/ewaV2YZ86tHt1goUXGYPP6Hn7k5lFSmRzRkL/hWdUGQ6dRddcQkSdF5JjL414ApdTHlVJbgIeAB+yXPQdsU0rdCvwx8Df67VwuUdabUEo9qJQ6qJQ6ODg4WM/3tYRWTDtLOIrOoLRlRC2eQWHaWalnoN+3Gc9AF521U0DW3LljA8+cnVo1M5G1Mehr0hgA3P+mnYCpMTB0F1V3DaXUW5VSB1weXyo69WHgPfZr5pRSC/bXjwF+ERnA8gS2OF6zGbjcku+kAqEyGUD1kEjr1FL3ugU9wayiZ6AHq6eWtjUAp2fgbbhr6XIJyGAZg4mFFGcmFtt+rVbQqjARWOm1b7lhiC0bepp+L4OhU2jqL0NE9iilXrb/+y7gJfv4CHBNKaVE5E4sozMJzAB7RGQHcAm4D/iZZtZQC2G/tynNQCll5aj7PI4itiLPIJlBhIoVrlrMdlYhay8g4NAMmg4TLZNnAPDM2Sl2Dfa1/XrNUvAMWtPe+9P/9HZWiVNkMNREs7dJ/0FE9gE54DzwYfv4e4FfEpEMEAfuU1Y8ISMiDwBfB7zAZ5VSx5tcQ1WaDRM5m8npcI6bZ9Dj9+LxlI8h67tS50yDRCaL3yt47dcFWyEgt7HoTLNzoJeBviDPnJ3ip+/c2vbrNUtB02lNaCfYYFsLg6FTaeovQyn1njLH/wT4kzLPPQY81sx16yUUcG8hUSv5WgC/I0xUFMqJpTIVawzAWWew1DNwehOt8AyCbRpu40REuMvWDVYD+TBRBU3HYFjLrIm8uLDf05RmkHCIvCG/e5hoMZmtWGMA1l2/RwqtK8AeeelIT2zGM0i3ebhNMXfu2MClmTgXp2PLcr1maKWAbDB0I2vEGDQnIGu9IeQrDK0v1iBiqUzFTCKw7qatmQZLi86cIYegz0uyyXYUyyEgw1LdoNNppYBsMHQja8IYhJoUkLVnEA54C55BUZhoMZmtmEmksdpYO+sMlnoGgTKN8GohlckhAr4KukUr2TccIRryrQpjsJDKEPB6lkVcNxhWI2viL6NZATmR1ww8Zdtb1OIZQOlMg1LPoAnNIKvwez3LVgjl8Ui+3qDTWUxmajLWBsNaZU0Yg2brDPRr9TwD61iRZpCq1TNYOtPAMgbFnkHjAnJwmUJEmjt3bODMxCLj88llvW69WJ6bCREZDOVYE8ag2TqDQpvp8qmlsWRjnkEinV1iDKyis8Y1g3b3JSrmwJg13+DUtc5uWreQzBjx2GCowNoxBulsw60TnGGiShXI1bKJwGpxvVgUJnL22A/6PKSaqEBejhoDJzvtgrMz4wvLet16WTTGwGCoyNowBgEvOdX40Jh8mMjvFJBLexOFa/EM/F7izjBRiWfgadgzSGVyyy6QDkeD9Aa8vDK+8m0p/vxbZ/jEl465PmdpBsYYGAzlWBPGoFycv1a0MQj7C6MpnWGnVCZHOqtq8wwC3iXzDFKZHEF/adFZI15MKptbtrRSjYiwY7C3I3oUfevlCZ48cd31ORMmMhgqsyaMQdjvHtqpFadn4PEIAd/SATe1TDnT9ASXZjaVagYelCp0IK2HVCbX9vbVbuwc6OuIMNFcIp2fTVxMram/BsNaZU0Yg1CZQrFa0SEh/T5hv3dJxo8WhGvzDEqziUL+pQIyNBbSSmeXP0wEsHOwl0sz8aY7wzbLXDzNfDLjOo5zwYSJDIaKrAljoD2DRmsNnKmlYBkF58ZXl2cQsIbj6A2ruM5Ab+aNFJ6tRJgILBFZKTg3ubKhotm49XPQ84k1SikWUyZMZDBUYk0Yg1CgWWNghV90R9Li0Ze1zD/W9BStJZkpDRNBZc8gl1P85Ge+y5MvXltyPJ1RKxQm6gXgzAqLyHO2EZgtChXFUlmUMq0oDIZKrAljkNcMGg0TFbWMCPm8S8ToxRqmnGn0OTE7nJHOqjKeQXljMB1L8ey5aQ6fn15yPLUCdQZghYlgZdNLE+lsvmvrXDyz5DnTl8hgqM6aMgbNhInCjoyfkL9IQK5h/rEm38Y6lS0ZeQkFzaBSeunUYgqA2XhqyfGVEpB7Aj5G+0Mr6hnMOUJDxZ5BqwfbGAzdyNowBk2HibJLC8OKKpob8QwWU5n83X9xOwoozCZwYzJvDJZuepaAvDID2ncM9PLKCqaXOr2B4s+lEMYznoHBUI6WGQMR+VciouxZx4jFp0TktIgcEZHbHed+QEReth8faNUayhFuus5gacZPyO9dUnQWq2H+sSavGaSyee8iWFSBDFScgzy5YBmDmdjSTW+lBGSwQkVnxhcarvJuFqdnMJco5xkYY2AwlKMlO4eIbAHeBrzqOPxOYI/9uB/4tH3uBuATwF3AncAnRGR9K9ZRjlCzYaLMUs8g7F/aZlrHpGvxDLTBWExlHRPU6vMMphatpnAlnsEKhYnAqjWYT2SYWEhVP7kNOOsLSj0DoxkYDNVo1c7xR8BvAM7bwnuBzymLp4F1IjIKvAN4Qik1pZSaBp4ADrVoHa7km8s1ISA7R1MWZxNpz6CnhmyisN/akOKpTGG2ss/NM6geJir1DNSKCMiw8iLybCVjkDLGwGCoRtM7h4i8C7iklHqh6KlNwAXH/y/ax8odd3vv+0XksIgcHh8fb3iNTXsG6VzVbKKAz1NTiCbvGSQLAnJx11KoTUAurrZNZbIr5hns0g3rVkg3mEtYG75HSj8XEyYyGKpT01+HiDwJjLg89XHg3wBvd3uZyzFV4XjpQaUeBB4EOHjwYMPBaL/Xg98rTQnIg5Fg/v9u2US11BhAQcyOpbN5g+KaWlpJM7CNwXwyY3cqtV6TzqoVm+Q1ti5MwOdZMc9AG4DR/nDZMFFfyBgDg6EcNf11KKXe6nZcRG4GdgAv2NO1NgPPicidWHf8WxynbwYu28ffXHT8H+pcd900M/qyNLW0NJuoFr0AChktsWSmTGppDZqBIy4/F0+zsc8yVKkVaGGt8XqEHRt7Vyy9dC6RJuDzMBgJuqSW2mE8v0ktNRjK0dRtpFLqqFJqSCm1XSm1HWujv10pdRV4FHi/nVV0NzCrlLoCfB14u4ist4Xjt9vH2krY3/i0s+JsoqDfGkCjM2didTRB00Yl5hCQ69UMdJgICvHxbE6RzSkC3pXb8HauYPfSuXiGaMhPNOzPh4w0i8kMvQFvvoLcYDCU0s6YwmPAGeA08GfALwMopaaATwLP2o/fsY+1lXCg8TnIxdlE2jDoDbsez8DjEcJ+LzGngFxn0dnkYorR/hAAM7YxSNvtK/wrVGcAljF4dSqWX8tyMpdIEw376A/7SzQDM8vAYKhOS/9CbO9Af62Aj5Q577PAZ1t57Wo05xkUp5Z6lxyP1Tj/WKNHX+bHadZRdJbLKaZjKe7euYErs4m8Z6B7Ga2UgAxWemk2p3h1KpYXlJeLuXiaaMhPf9jnWoFsxGODoTJrogIZ7Dh/A0VnSikrTOTYsIuH5SzWOP9Y0xP02u0odJ1B7QLybDxNNqfYYTeHm7XTS9P2e62UgAzO9NLlDxXNxdP0h/1EQ5Zn4Cx+M+2rDYbqrCFj4GmozqAQyikNE+k7+1iN8481PX6fHSYq9Qy8HsHvlbKegc4k2jlg3XkXewYrVYEMKzsPeS6RIRr20x/2k8mpfO0H6DCREY8NhkqsGWMQ9jemGThHXmp0AZpOL42lMjXNMtAUewZOARmsUE85zUCLxzvsu/CZvGeg8q9dKfrDfgb6AivmGURDlmYASwvPFpJZEyYyGKqwdoxBgwJyIl0ayskXsdl3n4t11BlAZc0ALC+knGegW1EMRYL0BX2lnsEKhonAHoE5sbyegVLKFpCtbCJY2p/ICMgGQ3XWjDFotM6gMP/YmVqqw0TWxLJ4OlufZhDwsZi0somcQ3M0lmfgvlYdJtrYG6Q/7GfGbk5VtBIAABLLSURBVGOtjcdKegagG9Ytr2eQSOdIZ5UtINueQcwYA4OhHtaMMWg0myiRLwwr9QwSmWze26g3myietuoMir0CsIxNWc/ALjhb32ttfHkBWWcTrWBqKcC2jb1MLqbyVb/LgfYCdGopFIeJTDaRwVCNNWUMmgsTFT4qrR8k01lidXQs1fQEvCwmrRbWzhoDTSXNYHIxRSToI+jzsq7H75JaurJC6XDUqoa+Pp9ctmvqz2CJZ2Afy2RzJDM5M8vAYKjC2jEGAcszqLfffj5M5HPxDNI5FuvoWKrpCfisrqXpXIl4DJZnUMkYbOgLALCux18oOsvobKKV9QyGIlYx3PW5xLJdUxeZ6dRSKDSuW6xjCp3BsJZZM8Yg5PeSU5UHzbuhvYlQwD21tJ5ZBpqegNdqVFfBM6gkIG/otYxBf7jUM1hpAXnI9gyuLaNnUAgT+YmEfIgUPIMFu311xDSpMxgqsqaMAUAiVZ8xSLp5Br5CS+x6ppxpegI+lLLuaF09A5+3vIC8kGJjr7Xh9ocDzMasAqtOEZCHV8QzsDb8aMiHxyNEgr68t2AG2xgMtbFmjEG4wZkGbprB0jBRY54BWDUDdQvIiyk2OjyDVDaXz6aBla1ABkvEDfg8y6oZOD0D/a82BgvGGBgMNbF2jEHA+lbrNwal2UR6A0+ks/l01XqziQCmF1NLjIymnICslNWXyKkZAMzEU6Sy2fxrVxIRYTgaXBHNQIeCnOGzRTPYxmCoibVjDIoKxWrFzRh4PELAZw24yYch6qwzAJiKpcoIyO5FZ3OJDOmsWuIZgFWFrCuQV1ozAEtEvja3nJ5BhpDfk/8s3YyBySYyGCqz8jvHMtHo6MtEpjRMBJZxSaZzdc0/1vQEC2EmtzBROc9At6LQAvI6RxplMtsZ2URgpZden18+z2A2ls5nEYGVYqpDR3qwjfEMDIbKrBlj4KwNqAe31FKwG9+lsw0NW3dO3HI2wCscczcGkwvW3XY+m6jH6RnYfY5WuM4ALM9guTUD7SVBGc/ApJYaDBVpiTEQkX8lIkpEBuz/v1lEZkXkefvxm45zD4nISRE5LSIfbcX1a0HPHq7XM4inswR8pS0jQnZFcyyZxSOl/YUq4TQcobKeQek6na0ooBAmmounO2K4jWYwEmQ+kWl4zGi96L5Emn5HMZ4RkA2G2mj6L0REtgBvA14teupbSqkfLTrXC/ypff5F4FkReVQp9WKz66hGo2GiZNEsg/z7+ayK5sVUht6AD3sGdE2EA07PwD2bqFKYaGNeQLb+nYmnOia1FGA4aqeXzifYtrG37debi2cYsD8TsFJME+kcyUyWhWQGn0fqMtYGw1qkFX8hfwT8BlBLae+dwGml1BmlVAp4BLi3BWuoSjMCcsgllGOFiXLEktm8BlArTjHTVUC2i86Kq6WLNYPegBevR5i1PQMRax7CSjMUsQvPlklELvEM8h5TJt+krh5jbTCsRZoyBiLyLuCSUuoFl6fvEZEXROSrInKTfWwTcMFxzkX7WNsp1Aa0xhgE7TCR9gzqYYln4FpnYD1fXC09uZCiN+DNr0dEWBf2MxOzBGS/19MRm57TM1gO9MhLTdQhrJsmdQZDbVT9KxGRJ4ERl6c+Dvwb4O0uzz0HbFNKLYjIDwN/A+wB3Haqsh6FiNwP3A+wdevWakutSKOaQSKdc60FCPu9zMTTxFL1ewY9S1pbuFUgF+YgOz2HqcVkvsZAo8XSoM9LsANCRLC8noE1yyBDNFz4VXbONDBTzgyG2qi6eyil3qqUOlD8AM4AO4AXROQcsBl4TkRGlFJzSqkF+/WPAX5bXL4IbHG8/WbgcoVrP6iUOqiUOjg4ONjwNwkFoTZeZzuKRKZ8mChp9yaqp/oYrNGUOrbvmlqan4Nc5Bkspthgi8caLZams7mOqDEAqxgu4PUsi2ewmMqSzamSbCKwPIPFZNaIxwZDDTS8eyiljiqlhpRS25VS27E2+tuVUldFZETseIWI3GlfZxJ4FtgjIjtEJADcBzza9HdRAz57A647myhVzhjY2UR1zj/WaG/CNUzk8AycOFtRaHSYKGUPyukERITBSJDxZfAM5hztqzXOLCsTJjIYaqNdu8d7gWMi8gLwKeA+ZZEBHgC+DpwAPq+UOt6mNZSgawPqIZHJuRsDRzZRPfOPNbrWwO29y3oGC6m8eKzRYSLLM1h5vUAzHA1ybRk8g+K+RFAwDHNxK0xkjIHBUJ2W/ZXY3oH++k+APylz3mPAY626bj3omQb1kExnCUWCJcd1NpHQqGdgffSuqaW2TuD0DJRSlmdQpBms6wkwE0vlBeROYSgS4pXx9s9CLnQsLRcmMiMvDYZa6JzdYxkINTDtrHxqaSGbqF7NAAoisltqqQ73OAvPFpIZUtlcSZgoGvYzn8yQTGc7JkwE1lyDa8vQrC4fJnIIyAGfh7Dfa7KJDIY66JzdYxkI+70N1Bm4ZxMF/V6SGas3USPZKgVj4F50BkvDRIUag6VeyrqwH6UscXml21c7GY6GmEtkGpo7XQ/5MJHDMwDLOMzG0yw2+PMxGNYanbN7LAMNeQZlsol0EVs2pxr0DHz5NRWj7/CdYaJCK4pSzQBgfD7ZUZ7BoB1au95mEbngGSw1Bv1hP+PzSbI5ZcJEBkMNdM7usQyE/fVrBol0Nr/xO3F6Cw1pBhU9A7upniNMNLWwtPpYo2cajM8nO0ozWK7Cs1nHlDMn/WE/l2esa5swkcFQnc7ZPZaBcKA+z0ApZbWZLqMZaBrKJqqgGbillha3otBoY5DMdE6dASxf4dlcIk1vwIuvyBBGQ34uz8QBM8vAYKiFztk9loF6NYNkmVkGxcca2Wx0mMgtm8gttXSyqEmdxlls1UlhouXyDObi6ZIQEVify7zpWGow1Ezn7B7LgJUBVHsFcrlZBsXH6m1HAQXPwO29g27GYCFJ2O8t0Sf6wwXjEOigOoP1PX78Xmn7XIO5RLpEPIalGoIJExkM1VlTxiAcqK/oTBuOcqmlmsY8AztMVKNnMLVYWnAGnesZiAiDfe1PL52LL+1LpHEaA5NNZDBUp3N2j2VAVw3XSmH+cfn0T6hv5KVmpD9MyO9xvWt1KzqbdCk4A8tw6Ot3koAMMBQNMb5CnkG/8QwMhrpYU38lWkBWStXU6jmR0cagfGopNGYMfvy2MV6/e6NrPLsQJnJkE5UxBmBtfLFUtqMEZLBE5HOTi229xlwizd7hSMnx/iWewZr6NTcYGqKzdo82E/J7Uaq05085tNjsnlrqCBM1sNn4vB5G+8Ouz+UrkO0wVTKT5dWpGMORkOv5euPrpDARWCJyuzWD2djS+ccaZ6qpMQYGQ3U6a/doM3pTr1U30JqBW1w/1KRnUAmPRwh4PfnhNl87dpXZeJofuWXU9XydXtpJFchgeQYzsXTbqpBzOcV8MlNSYwBFnkGLfz4GQzfSWbtHm6l3wE2lMFFoiWbQ+jvPgM+T9wweevpVtm3s4Q27B1zP7WTPAGibbrCQyqBUafUxWHMewLoBKK5BMBgMpaypv5J65yAna0gtDfk9bZk7HPR5SGWznLw6zzPnpnjfXVvxlLnOOju9tNME5MGo3ZKiTbUGbrMMNNpAmhCRwVAbnbV7tJnCHOTaNINCamn5MFG7qlu1Z/Dw984T8Hl47x1byp6r74I7aZ4BkNc42tWfKN++2i211DYQfSat1GCoiTV121R3mChdPkykM34aKTirhaDPw3QszdeOXeVHbh51rTHQdGqYaCjvGbTJGJTpWAqWjuPziPEMDIYaaWr3EJHfEpFLIvK8/fhhx3MfE5HTInJSRN7hOH7IPnZaRD7azPXrRc9Brl1ALp9N5PEIQZ+nrZ7BN05dZz6Z4X13ba14bt4YdJiAvKEngM8jbSs8K9exFKyit/6w3xgDg6FGWvGX8kdKqf/kPCAi+7HmG98EjAFPishe++k/Bd6GNTP5WRF5VCn1YgvWUZW8Z1CjZhCvUIGsj7c6k0gT9HlJZxU3jES4Y9v6iufms4k6zDPweKxZyO3yDGZtY+CWWgqWkTAFZwZDbbTrL+Ve4BGlVBI4KyKngTvt504rpc4AiMgj9rnLYwz8jYWJ3NpMg6UltOvOU9/lv++urVUL5PRm2GkCMljppW3zDBKlIy+d/PO37K4YXjMYDAVasZM9ICLvBw4Dv66UmgY2AU87zrloHwO4UHT8rhasoSb0Hf6//8qLfOrvXq56/sRCkoDPUzaLJ+z3uoaQWkHQbjPx46/ZVPVcnU3UaWEisFpSfPPUOG/7w2+0/L11W+8+lzoDgHffvrnl1zQYupWqxkBEngRGXJ76OPBp4JOAsv/9A+BDgNvuqXDXKFSFa98P3A+wdWvluHktbFoX5oOv215zquOe4T5uGusv+/y/+Md78rn0rebnXr+DxWSGSJm7Xic3jEb48A/tKluHsJL87N3b8Hvbl+W0dzjSltReg2GtIUqV3YvreyOR7cCXlVIHRORjAEqp/9d+7uvAb9mn/pZS6h328SXnVeLgwYPq8OHDLVmrwWAwrAVE5PtKqYO1nNtsNpGzP8JPAMfsrx8F7hORoIjsAPYAzwDPAntEZIeIBLBE5kebWYPBYDAYmqdZzeA/ishtWKGec8A/A1BKHReRz2MJwxngI0qpLICIPAB8HfACn1VKHW9yDQaDwWBokpaFidqNCRMZDAZDfSxbmMhgMBgM3YExBgaDwWAwxsBgMBgMxhgYDAaDAWMMDAaDwcAqyiYSkXHgfIMvHwAmWricVtGp64LOXVunrgs6d22dui7o3LV16rqgvrVtU0oN1nLiqjEGzSAih2tNr1pOOnVd0Llr69R1QeeurVPXBZ27tk5dF7RvbSZMZDAYDAZjDAwGg8GwdozBgyu9gDJ06rqgc9fWqeuCzl1bp64LOndtnbouaNPa1oRmYDAYDIbKrBXPwGAwGAwV6GpjICKHROSkiJwWkY+u8Fo+KyLXReSY49gGEXlCRF62/6087Lg969oiIn8vIidE5LiI/EoHrS0kIs+IyAv22n7bPr5DRL5nr+1/2u3Qlx0R8YrID0Tkyx22rnMiclREnheRw/axTvh5rhOR/yUiL9m/b/d0yLr22Z+VfsyJyL/skLX9qv27f0xE/of9N9GW37OuNQYi4gX+FHgnsB/4aRHZv4JL+ivgUNGxjwJ/p5TaA/yd/f/lJoM1rvRG4G7gI/bn1AlrSwJvUUrdCtwGHBKRu4HfA/7IXts08PMrsDaAXwFOOP7fKesC+EdKqdscKYid8PP8L8DXlFI3ALdifXYrvi6l1En7s7oNuAOIAV9c6bWJyCbgXwAHlVIHsNr+30e7fs+UUl35AO4Bvu74/8eAj63wmrYDxxz/PwmM2l+PAic74HP7EvC2Tlsb0AM8hzUzewLwuf2cl3E9m7E2iLcAX8Ya9bri67KvfQ4YKDq2oj9PIAqcxdYpO2VdLut8O/CdTlgb1tz4/9ve+YREFUVh/HegkrTIjArCoISINpEuJDIiKIgkbNOiaOGiZZtWQQRB+wh3bYoWEQaVhLgq+rO1siwMoz8UappKYEErq6/FPYODKLRovI/h/ODx7r28xcc7ZzhzvzvDGQUaSL1n+oBDlcqzqt0ZMPciS4z5WpHYKGkCwO8bcorx1qXNQD8F0eZWzCAwBTwAPgIzkn75I7ni2gWcBf74fF1BdEFqNnXfzAa8jzjkj2cTMA1cd2vtqpnVFUDXfI4D3T7Oqk3SF+ASMAJMAN+BASqUZ9VcDBbqkh4/nVoEM1sF3AXOSPqRW08JSb+Vtu+NQCuwY6HHllKTmR0BpiQNlC8v8GiufGuT1EKySE+b2b5MOspZBrQAVyQ1Az/JY1UtinvvHcDt3FoA/IziKLAV2ATUkWI6n/+SZ9VcDMaAzWXzRmA8k5bFmCz1kfb7VA4RZracVAhuSuopkrYSkmaAJ6RzjXozK7VszRHXNqDDzD4Dt0hWUVcBdAEgadzvUyTvu5X88RwDxiT1+/wOqTjk1lXOYeCFpEmf59Z2EPgkaVrSLNAD7KFCeVbNxeAZsM1P3leQtn+9mTXNpxfo9HEnya9fUszMgGvAsKTLBdO23szqfbyS9OEYBh4Dx3Jpk3ROUqOkLaS8eiTpZG5dAGZWZ2arS2OSBz5E5nhK+gqMmtl2XzpA6pGePc/KOMGcRQT5tY0Au82s1j+npXdWmTzLeVizBAcw7cA7ks98PrOWbpLvN0v6lnSK5DM/BN77vSGDrr2kbeZrYNCv9oJo2wm8dG1DwAVfbwKeAh9IW/qajHHdD/QVRZdreOXXm1LeFySeu4DnHs97wNoi6HJttcA3YE3ZWnZtwEXgref/DaCmUnkW/0AOgiAIqtomCoIgCP6RKAZBEARBFIMgCIIgikEQBEFAFIMgCIKAKAZBEAQBUQyCIAgCohgEQRAEwF+2syCHfohbhAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "actor_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.0005}\n",
    "critic_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.0005}\n",
    "agent = OffPACAgent(env, actor_kwargs=actor_kwargs,\n",
    "        critic_kwargs=critic_kwargs)\n",
    "behavior_agent = RandomAgent(env)\n",
    "\n",
    "# 训练\n",
    "episodes = 80\n",
    "episode_rewards = []\n",
    "for episode in range(episodes):\n",
    "    observation = env.reset()\n",
    "    episode_reward = 0.\n",
    "    while True:\n",
    "        action, behavior = behavior_agent.decide(observation)\n",
    "        next_observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        agent.learn(observation, action, behavior, reward,\n",
    "                next_observation, done)\n",
    "        if done:\n",
    "            break\n",
    "        observation = next_observation\n",
    "    \n",
    "    # 跟踪监控\n",
    "    episode_reward = play_qlearning(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "# 测试\n",
    "episode_rewards = [play_qlearning(env, agent) for _ in range(100)]\n",
    "print('平均回合奖励 = {} / {} = {}'.format(sum(episode_rewards),\n",
    "        len(episode_rewards), np.mean(episode_rewards)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 柔性执行者/评论者算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNReplayer:\n",
    "    def __init__(self, capacity):\n",
    "        self.memory = pd.DataFrame(index=range(capacity),\n",
    "                columns=['observation', 'action', 'reward',\n",
    "                'next_observation', 'done'])\n",
    "        self.i = 0\n",
    "        self.count = 0\n",
    "        self.capacity = capacity\n",
    "    \n",
    "    def store(self, *args):\n",
    "        self.memory.loc[self.i] = args\n",
    "        self.i = (self.i + 1) % self.capacity\n",
    "        self.count = min(self.count + 1, self.capacity)\n",
    "        \n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.count, size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SACAgent(QActorCriticAgent):\n",
    "    def __init__(self, env, actor_kwargs, critic_kwargs,\n",
    "            gamma=0.99, alpha=0.2, net_learning_rate=0.1,\n",
    "            replayer_capacity=1000, batches=1, batch_size=64):\n",
    "        observation_dim = env.observation_space.shape[0]\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = gamma\n",
    "        self.alpha = alpha\n",
    "        self.net_learning_rate = net_learning_rate\n",
    "        \n",
    "        self.batches = batches\n",
    "        self.batch_size = batch_size\n",
    "        self.replayer = DQNReplayer(replayer_capacity)\n",
    "        \n",
    "        def sac_loss(y_true, y_pred):\n",
    "            \"\"\" y_true 是 Q(*, action_n), y_pred 是 pi(*, action_n) \"\"\"\n",
    "            qs = alpha * tf.math.xlogy(y_pred, y_pred) - y_pred * y_true\n",
    "            return tf.reduce_sum(qs, axis=-1)\n",
    "        \n",
    "        self.actor_net = self.build_network(input_size=observation_dim,\n",
    "                output_size=self.action_n, output_activation=tf.nn.softmax,\n",
    "                loss=sac_loss, **actor_kwargs)\n",
    "        self.q0_net = self.build_network(input_size=observation_dim,\n",
    "                output_size=self.action_n, **critic_kwargs)\n",
    "        self.q1_net = self.build_network(input_size=observation_dim,\n",
    "                output_size=self.action_n, **critic_kwargs)\n",
    "        self.v_evaluate_net = self.build_network(\n",
    "                input_size=observation_dim, output_size=1, **critic_kwargs)\n",
    "        self.v_target_net = self.build_network(\n",
    "                input_size=observation_dim, output_size=1, **critic_kwargs)\n",
    "        \n",
    "        self.update_target_net(self.v_target_net, self.v_evaluate_net)\n",
    "        \n",
    "    def update_target_net(self, target_net, evaluate_net, learning_rate=1.):\n",
    "        target_weights = target_net.get_weights()\n",
    "        evaluate_weights = evaluate_net.get_weights()\n",
    "        average_weights = [(1. - learning_rate) * t + learning_rate * e\n",
    "                for t, e in zip(target_weights, evaluate_weights)]\n",
    "        target_net.set_weights(average_weights)\n",
    "        \n",
    "    def learn(self, observation, action, reward, next_observation, done):\n",
    "        self.replayer.store(observation, action, reward, next_observation,\n",
    "                done)\n",
    "        \n",
    "        if done:\n",
    "            for batch in range(self.batches):\n",
    "                # 经验回放\n",
    "                observations, actions, rewards, next_observations, \\\n",
    "                        dones = self.replayer.sample(self.batch_size)\n",
    "                \n",
    "                pis = self.actor_net.predict(observations)\n",
    "                q0s = self.q0_net.predict(observations)\n",
    "                q1s = self.q1_net.predict(observations)\n",
    "                \n",
    "                # 训练执行者\n",
    "                self.actor_net.fit(observations, q0s, verbose=0)\n",
    "                \n",
    "                # 训练评论者\n",
    "                q01s = np.minimum(q0s, q1s)\n",
    "                entropic_q01s = pis * q01s - self.alpha * \\\n",
    "                        scipy.special.xlogy(pis, pis)\n",
    "                v_targets = entropic_q01s.sum(axis=-1)\n",
    "                self.v_evaluate_net.fit(observations, v_targets, verbose=0)\n",
    "                \n",
    "                next_vs = self.v_target_net.predict(next_observations)\n",
    "                q_targets = rewards + self.gamma * (1. - dones) * \\\n",
    "                         next_vs[:, 0]\n",
    "                q0s[range(self.batch_size), actions] = q_targets\n",
    "                q1s[range(self.batch_size), actions] = q_targets\n",
    "                self.q0_net.fit(observations, q0s, verbose=0)\n",
    "                self.q1_net.fit(observations, q1s, verbose=0)\n",
    "                \n",
    "                # 更新目标网络\n",
    "                self.update_target_net(self.v_target_net,\n",
    "                        self.v_evaluate_net, self.net_learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均回合奖励 = -9790.0 / 100 = -97.9\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXmcHGW5739v9Tr7mmSSTCaZLCRkIQmEBBRkEdmOCLhdcEOP5yAK6vVcvYres7mco8dzXLgiHkQ86kXRoyIIyCZbkCUQCGQnyWSZSWYymX3p6aWq3vtH1VtdXV3VXb1N9Uw/389nPpmp7umqTk+9z/s8v2dhnHMQBEEQlYvk9QUQBEEQ3kKGgCAIosIhQ0AQBFHhkCEgCIKocMgQEARBVDhkCAiCICocMgQEQRAVDhkCgiCICocMAUEQRIXj9/oC3NDa2sqXLFni9WUQBEHMKLZv3z7AOZ+T7XkzwhAsWbIEr7zyiteXQRAEMaNgjB118zwKDREEQVQ4ZAgIgiAqHDIEBEEQFQ4ZAoIgiAqHDAFBEESFQ4aAIAiiwiFDQBAEUeGQISCIMuKp/f04MjDp9WUQHrL96BB29oxO6znJEBBEmcA5x833vIrvPP6m15dSkcRlFZMx2dNriCYUfPxnr+D6H7+Iw9O4ISBDQBBlwsBEHJG4gp3Hp3c3WOkMTMTwvSfexFu++Wdc+t1nMRVXPLuWR3f3YSSSQFxRcfM9ryKamJ5rIUNAlCUxWcGuClsQu4cjAIDDA5MYiyY8vpryJSYruOWXr+Jg/3jBr/Wbl7vxlm8+ie89cQBL59Ti+MgUfvHikcIvMk/ueekYFrdU444Pnok9vWP46oN7puW8ZAiIsuT3rx7HVT94DodOTXh9KdNG91DE+H738TEPr6S86R6awoNv9OKurYcLfq17th3D4uZqPPF3F+A3nzgXbzttDu54+hDGPTDEB/snsO3wEK47uwNvP30ebrpgGX750jHcv+N4yc9NhoAoS7pOTYBz4LHdJ4v6uh/96Tb86JlDRX3NYmE2BDPZG3ruwAD+34tH0TMcyf5kHc656+dG4loc/6GdvQWFTjjnONQ/gXOXtWD53FoAwOcvPQ3DkQR+8lyqkVFV99eXL/duOwa/xPDes9qNa9m0uAl3PttV8vPPiO6jROXRPTQFAHh8Tx8+eeGyorxm16kJPL3/FLqHIrjpguK8ZjHpHprCnLoQ/BKbsTrBY7v78Kl7XoWsL1wr59XhpguX4tqN7Y6/c99rPfj6g3vx3BcvRlXQl/UcET2GPx6V8fT+fly+dn5e19o3FsVETMYK3QgAwBntjbh8TRvu2noYN5y7BIOTcfzzH3fjxa5BbFrcjItXzcWla+ZhcUtNXud0IppQ8LtXe3DZmjbMqQsBAPw+CT/80JmoCvggSayo57NCHgFRlvSMaLvJ17pHcGo8VpTXfGKv5l0cOjWJEyNTRXnNYnJsKIJFTVVYu7BhRnoET+3vx82/fBVrFjbgwU+fh//zV6dD4Rx//4fdkBXV9nc45/jPZ7owOBlH/3jU1XmERwAAf3jtRN7Xe7BfCzsuMxkCAPi7S0/DZFzGh+9+CZd/71ns6B7Be89ahKHJOL7x8F6847vPFkWfMPPo7j4MRxK4fnNHyvG5dWHUhQNFPZcdZAiIsqRneApnLW4C58Cf9xYnPPT4npNoqtZuqucODBTlNYtJ93AEi5qrsW5hA7oGJj2JU+fL8wcHcNMvtuO0eXX4+cc2Y+3CBvzN+UvxmbevwERMxt5e+4Xz9Z5R7OvTHhuajLs612RM8wjOX9GKJ/f1YzSS3/+TMATLLYbgtHl1ePfGduw+MYb3nNmOpz5/If713evw6Ofehqc+fyF8jOH2p4oXXlRVjp89fwQdzdV4y7KWor1uLpAhIMqO8WgCI5EELjl9HhY2VuHxPYUbgsGJGLYfHcaHz1mMOXUhbD1YXoYgoajoHY2iQzcEALD7RHEE4+1Hh/D+H72QspMuJqrK8YXfvoFFzdX4xce3oKE6uYPdvKQZAPDS4UHb3/3VS8eM790aApHe+cEtHYgrKv60qxeAprF8+Ccvud44HOyfQENVAHNqQ2mPfePatXj2CxfhW+89A62mxztba/Chczpw/47jODpYnDz/O545hFePjeDmi5aVPATkBBkCouw4rodtFjVX4R2r5+G5gwMFL2J/3tcPlQOXrmnD+ctb8ZeDA9MiALqldyQKReVY1FSNtbohKFZ46Dcv92DbkSHs6B4pyutZefnIEI6PTOGWi5ajuSaY8lhbQxiLW6qx7fBQ2u+NRxN44PUTeOtybRfs2iPQ/xa2dLZgaWsN/rDjOHYdH8W773geWw8M4P4d7sJFB/onsHxuLRhLX3zDAR8WNVfb/t7fnr8Ufp+EO54u3Ct4qWsQ//HYfly1fgHev2lRwa+XL2QIZgkvdQ3iTzt7S/b6nHP86JlD0xJb79GF4vamaly6eh5isopn3yxsB//4npNY0BDGmgX1OP+0VgxNxrGnt3xSNEUNwaLmasypC6GtPlwUwZhzjmfePAUAeL27NLrDH3acQHXQh0vXzLN9fPOSZrx8ZCjN8P5hxwlMJRTcfOFyAMBwxJ0hEGJxdciHqzcsxItdQ/gf//kCgj4J6xY2YPeJ9Pc5GZPTMowO9U9g+ZzatOdmY259GNedvQi/e7XH2LS8eXIc//7o/pw2LAMTMXz6V69hcUsN/vXd62wN0nRBhmCWcNdzh/GtR/aV7PWPDEbwzT/tw8MlNDYCsSi2N1Xh7M5m1If9OYeHuocixo0/FVew9cApXLJ6HhhjeOvyVgDA1iLpBIrKCxYPjw0JQ1AFAFi7sKEohmD/yXH0jWki7Osl8AhisoKH3jiBS1fPQ3XQPglxc2czhiMJHDTVhHDO8cuXjmH1/Hqcu6wFQZ+EoUl3sf5IXIZfYgj6JFyzcQEkBnS01OD3n3oLLlo1F10Dk2kL8gd+/CJu/f1O4+ehyTgGJ+NYMS93QwAAn7hgGTgHvvv4m/jnP+7GFd/fih88dRBP7Tvl6vdHIwnc8stXMTqVwO0fOBO1IW8TOMkQzBIicRkjU6UTF7v0m3g8WvpeLD3DUwgHJLTUBBHwSbho1Vw8ue+kY+aJlVPjMbz9O8/gqv/7HPb3jeO5gwOIJlS8Y7W2Y51bF8aqtjpsPeDups3G77b34JLvPIs/vp5/Bkv3UAR+iWF+g2YI1i1swOGBSUwU2Pvmmf3ae9zS2VyU0NAbPSMpIZyn95/CWFTG1RsXOv7Olk4t9POSKTz0Rs8o9vaO4QNbOsAYQ1NNAMM5iMVVQR8YY1jcUoM/ffZt+N0nz8W8es3j4xzY35c0zCOROF7vGcVju/sQl7W/IaeMIbcsbKzCe85sx2+39+C/nj+C929qB2PJ183E9qPDuPK2rXjlyDC++Z51WL2gPq9rKCYFGQLG2PsYY7sZYypjbJPlsVsZYwcZY/sZY5eZjl+uHzvIGPtSIecnkkTiCsamEiWLe4sK3+loytUzHEF7U7XhKl+xdj6GIwn8+pVuV7//2B7thu8fj+FdP3gO//HYftSF/MaCBGgZJ68cGS5KX5nH9vQBAG79/c6URmGRuIz7dxx3VfTUPTyFhU1V8Oli4bp2bUHbfXwU+/vG8fH/ehk/ePJAztf2zJunsKqtDpevbUPfWBR9o/YpmpxzPLyzFzHZ+VpVleP6O1/E9Xe+aGQ03b/jOFpqgjhf97LsWNRchbb6cIpO8MOnD6Im6MPVGxYAAJqqgxhyGRqaiiuoMXkfK9vqDG9k9XxtUTWH/V47phnAybhiXIORMZRHaEjwuXechg9s6cAfbzkP//ruM9DeVJXi9dhx19YuvP8/XwBjwH/fdG7G+orppFCPYBeAdwN41nyQMbYawHUA1gC4HMAPGWM+xpgPwO0ArgCwGsD1+nOJApmKK1A5MF6ihbrrlLbATZYo88RMz/AUFjVVGT9funoezlveim88tBfHBrNXqz6yqw+drTV4/O/ehs2dzdjXN44LV81F0J/8cz9/xRzEFRXPvHkK97x0FJd+9xl86p7tOV9rNKHguYMDuHxNG/w+hk/pjcL29Y3hXT/4Cz577w788KmDWV9HqyFIipNCMP7qg3tw5W1b8ed9/TmHxyZiMl4+MoQLTpuD9YsaAQCv99h7BdsOD+FT97yKP+3sc3y9E6NTmIwr2H9yHJ/+1WsYnozjib39uGr9Avh9zksJYwybO5ux7fAgOOd44dAgHt19Ep+8cJmRI99cE8xJLK52KDxrb6pCfdifknG1/egwfBJD0C/hyX39ADRDUBXwYWFjle3ruKGtIYx/uXad8Vktn1Ob0SN4sWsQX39oL96+ai4e+sz52NjRlPe5i01BhoBzvpdzvt/moasB3Ms5j3HODwM4CGCz/nWQc97FOY8DuFd/LlEgQkAbK1F4SHgEE7HSd0PsGZ5Cu2lRlCSGb733DPgYwxd++3pGr2ckEscLhwZx2Zo2zK0L42cf24zvX7cBX7x8ZcrzNnc2I+iXcNP/246v3LcLo1MJPLyzD88fyk03eKFrENGEiuu3dOC779+Avb1j+Mjd23D1D/6CkUgCm5c0486tXVlF9p6hiKEPAFr4akFDGHt6x3Dd2Ytw8aq5GHS5UBrXdmgQCYXjgpVzsHp+PfwSc9QJRNimK8OO9siAZoSv2bAAT+8/hevufBFxWcU1GcJCgs2dzTg5FsPRwQi+/tAeLGgI42/OX2o83lwTdB0amoorqA7ZGwLGGFYvqMceiyFYPb8eb1nWgif3acb0QP84ls2tKWq65vK5tTh0agKKzd+nqnLjfd92/UY0VJW+SCwXSqURLARg9uN79GNOx4kCEYZgJM/immwYHkGJQ0Nj0QRGpxJob0rdqS1srMLfX7UaLx0ewn89f8Tx95/Y2w9Z5bhibRsAzYhcvWFhimEBtPTAj711Ca5Y24Zf33gOnvnCRWirD+PfH92fU9+bJ/f2oyrgw5bOZly0ai5uumAZth0ewubOZvzps+fjO/9jPTgH/i2DkD8ZkzE4GU9LV7zrhrPxyGffhm9cuw5LWmpc75gFz7zZj+qgD5sWNyMc8OH0+fWOOoEImRzO4HEd1vPmv3jFKnz8vE7sPzmOJS3VWN/ekPVatnRq9QRfvm8ndp8YwxevWIVwILmYN9e4Dw1NxmVUB5zF1dXzG7CvbwyKyiErKnZ0j+CsxU24eNVcHBmMoOvUBA71T2DF3DpX53PLirl1iMuqbY+l373ag13H0993uZBVqmaMPQGgzeahr3DO73f6NZtjHPaGx/auY4zdCOBGAOjo6LB7CmFiSg/ZjJbAIxiJxI3daKHiZTaODydTR62876x2PLqrD996ZB+uWNdmCKtmHtnViwUNYZzhYnG69YrTU37+zNtX4Mv37cRT+/tx8ap5espsF+7fcRzCNqyYV4vvvH8Dgn4JnHM8ua8f561oNW7uL1y2EpeumYcN7Y3GbvNvzu/E7U8dwkff2okNeojGjJE6annPZhGxpTaISFxBNKG4Wkg453h6/ym8ZVmrERLbsKgR9712HKrKU3bCCUXF9qPDAJBxOtqRgUmEAxLm1YXx5StPh19i2NjR5CrtcfncWjTXBPH8oUFsWNSId61fkPJ4U3UQo1MJyIqaMcwEaJsea72CmdUL6hFNqDg8MIFoQsVUQsGZi5uwcVEjgN344+u9ODEaTasoLhQhPB84OZHSiygSl/HtR/dj/aJGXHXGAqdf95SsHgHn/BLO+VqbLycjAGg7fXN1RDuAExmO2533Ts75Js75pjlz5mR/JxUM5xwRXZAcmcpt1+iGQ7o3EPRJJfcIRAdOq0cAaG7/Jy5Yhpis4sDJ9BDGREzGswcGcNnatrxyst+3qR2LW6rx7UffRFxWcevvd+Jbj+xDbciPztYaLGyqwoNv9OLHW7sAAG+enMDxkSlcvGqu8Ro+ieHMjqaUhfaTFy7HnLoQvv7gHltvQzTYcypgAmAsfG7DQ10Dk+gZnsIFK5P3zvpFjZiIyegaSP2/23V8FFMJBfPqQzgyMOnoER0ZmMSSFi2c4pMYbr3ydFy+1m6PmA5jDGcv0WLif//O1WmfT3NNEJy728hELGKxFSEY7z4xhlePaQburMVNWNRcjdPm1eLnLxwBACwrQCi2QxgWq2D8o2e60D8ewz+883TPKoezUarQ0AMArmOMhRhjnQBWANgG4GUAKxhjnYyxIDRB+YESXUPFEJNVY8daitCQiBufPr+u5B5Bj+ER2It4IrZqdx1P7etHXFZxRZ7dKAM+CZ+75DTs7R3Dlbdtxb0vd+PTFy/Hf990Ln704bNw90fPxhVr23Dbnw/g2GDEEB4vWjk34+vWhvz4/KWn4ZWjw3h6f3rKqqgh6HBhCNzG0f+it9C48LSkIdiwSPOSdlgKy0RY6D1ntmNcD1PZcXhQMwT58pm3r8C/vfcMnLU4XSRtEu/PRXgoEpMzdildPrcWQZ+EPb1j2H50GPPqQ1jQEAYAXGTSWvKtIXCioSqAuXWhFMF4NJLAnc8ewjvPmI+zFjcX9XzFpND00WsZYz0AzgXwEGPsUQDgnO8G8BsAewA8AuBmzrnCOZcB3ALgUQB7AfxGfy5RABFTCmQpQkOHTk0i4GNY1VZfFI/gpa5B/PrlY7aP9QxPoTroc3T9a3SRcMKmnuGRXX1orQ3aLjRuuWr9AqycV4euUxP4l2vX4X9dujJl9/oPV62GX2L4hwd24cl9J7FmQT3a9EUmE9dubEdVwGdU+ZrpHoqgJugzGuLZ0ZKjR3B4YBI1QV+KQV3aWou6kB87uodTnrvt8BCWzqnB2XpfILvwkKyo6B6KYElr/oZgzYIGxzYKzdX6+5twYQgSCmoyGIKgX8KKebXYc0IzBGctToav3r5KqyUJ+BgWZzC8+bJ8bi0OmAzB02/2I5pQ8dfndRb9XMWkoHI2zvl9AO5zeOwbAL5hc/xhAA8Xcl4iFXMVZSkMQdcpLebZUB0oikfw462H8dzBU3j3me0IWOLBWg1BlWNopy5k7xFEEwqe2t+PazYuNHLx88EnMdz9sbMxPBk30gLNzG+owufecRq+/tBeAMBnLl7u6nWDfglnLm607bnTo3cdzRTOEoZxaDK1JffoVAKTMRkLLGmQvSNRzG9M/X+UJIYzFjWktJpQVI5tR4bwzjPmG4v84YFJbFqSuns9MRJFQuHobC3+4gmYPB5XHoGCqgyhIUALDz20sxeRuIKPvmWJcfzMjkZj555Ni8iH5XNr8ftXj4NzDsYYntzXj5aaINa3p2tD5QRVFs8CzEVR+bbkzcShUxNY2lqDmqAf0YTqusLXCa39g5pS/Smwpo5aMTwCiyHoG40iEldwZhFysxc2VtkaAcFH37IEp+tx6ItWZQ4Lmdm8pAV7+8bSjPWxoUhGfQAAWmq0DpjWHfO/PbIPH/jxi2nP7x2dwnwbT2V9eyP29o4ZRW77+8YxHpWxubMZ7XpB2xGbrpoiY6iQ0FAmkoYu899vQlERV9SMHgGgCcbCUzZ7iH6fhM9fthIfL9EOffncWkzEZJwci0HW61QuWDmnoM3JdECGYBZgDg0VWyyWFRXHhiJYNrfWWIQnC6jG5ZwbMfHXjg2nPS48Aif8PglVAV+aIRCtL+rDpe/Z4vdJ+P51G/DJC5fltNPb3NkMzrW20AIt5DKVljFkpb7KD7/E0lJIjwxO4uhQJK0i+MRoFAtssqq2LG2BrHJ8/89alfI2vT305s4WBHwSFjVVGfUCKefRw0WFhIYy0aiHxbJ5BOJvPdskszULNEMe9EvG94IPn7MY120uTSaiEIwP9I/jte4RjEQSRjiqnCFDMAsQNwdjxQ8NdQ9PIaFwLG2tMRpjFaITDEzEMaXvRkXpv2B0KoGxqJzREABAbdif1vNItDyYjmlOgDa85IuXr8opC2RjRyMCPpbSc+e5gwOYSijYsjSzkKj140mvvu0djYJzLRQkiMsqBiZimN+Y7hG8bUUrrt/cgTuePoT/fqUb244MYWFjlVFhu6S1xt4jGJhEddCHuXXpvfuLQTjgQ03Ql7VWQoRBa7I0aVs1X6sRWN/ekFJRXmqMzKH+CTy5rx9+ieH805zbb5QLNLN4FjCV0G6OObWhomcNHTI15xI5/oUYAuEN1Ib8eM1S3NRjdB3NvDuuDfnTPQL957pp8AjyJRzwYX17qk5w/44TaKgK4MKV2VOkW2qCaWLxSb13UM/wlLFbPzmmGQc7j4Axhq9evQbdQxF8+b6dCPl9uHR1cse6pKUGLx8eMmLcgqODk1jcUlPSVsl2hs6K0YI6i0dQHw7gynVtRqfZ6WJObQj1YT8O9k9g+9FhnL2kGfXTtDkpBPIIZgHi5pjfEC56iwmRc76stRa1+iJbSD8jUSdw2Zo2HB6YTEmHzJY6KqgN+dOMUTI0VN433ebOZuzsGUUkLiMSl/Ho7j5cuW4+Qv7sRWJN1akL5Xg0YYTpuk3VrKKdhZ1HAGhpsrd/8Ex0NFdjIqbpA4LO1hpMxhWcmkgVpY8MRkomFAtaLIaAc542WyASE4Ygu8H/4QfPwge3LC7uRWaBMYYV8+qw9cAA9vWNp9SYlDNkCGYBSUNQVfRW1If6J9FaG0RDdaAooSHhEVy1Xsv1N7c8EKP/XHkEjqGh8vUIAM0QyCrHa8dG8Piek4jEFVyzwV21aXNt6kJ5ciwZDjK3NejVvQS7ymtBQ1UAP/3oZrxr/QKjPTeQ1ADMOoGROloioVjQVBNM0QiePTCAv7rtuZS+QUZoKItH4CXL59Qaf+cXn06GgJgmRNbQ/MYwInHF6LleDLoGJrC0VYt7imrOQg3BvPoQNnc2Q2JJwZhzjvt3nMBp82oz5tMDukbg4BHUlrkhOGtxEySmNXm777XjWNhYZeTvZ6OlJohB0069bzT5vfCmAK1LKADbrCEzHS3VuO36jWgxzeRd0qIZYXMtQc/wFGSVl0woFjRbPJ439E1C72jyvbkVi71E6ASLW6qxtMT/Z8WCDMEswBwaAgoTjE+MTGH9Pz+GT/ziFWw7PISuU5NYNlf7YxYeQSEdSI8NRdDRXI3qoB+r2uoNneDVY8PYfWIMHzl3SdY4tKYRpL7H8WgC4YCUVpdQbtSFA1izoAGP7e7D1gMDeNeGBa4F5+aaIMaiMhJ6+q6YPLagIZxiCHpHoqgP+7MKqnYsbKyCX2JGuiiQTB3tLPGi1mTpQLrvpJZebNa9Jl2KxV6yXK9YvnjVXE/HT+ZCed81hCsicRmMAfPqhSHIP4W0eyiC0akEntzXj/f/5wsYnIwnPQKRPlqgRiBy5jd2NGLHsRGoKsfPnj+KurAf17poaWwfGpKnLWOoUMSMBEXluGaD++a7LZaiKxEaOmtJsyU0NJVWYOYWv09CR3N1ikdgpI6WODTUXBPEpN5YDwD26cNlzOFOwyMoww6ego2LGrFuYQPee1Z5DJ1xAxmCWUAkrqAq4EOjXqZfiEcgeqnf+ZFN+Po1a3He8lYjzlljeAT5GYJoQkHfWNToqbOxownjMRkvdA3i4Z29eN9Zi1zt9GrDfkxavJLxmFz2+oBAiLOnz6/Hyjb3rZCb9aIyET7pHZ1CQ1UAy+fU4uRYzKglODESzRoWysSS1pqUSWtHBiZRG/Kjtda542cxaKpOGrpoQjGuYdSkG0Ri5e8RNFYH8cdPn5dWv1DOlO//JuGaSFxBddBvNGQrJIU0oRuC+rAfF52zGB86J5l1EfJL8Essb4/g+MgUOIfJEGjFWH//h12QVY6PnOsuw6M25EdcURGTFSPbZkZ5BEuaURXw4f2bctsxGtW3enVx32gM8xvCxkCb48NTWDqnFn1jUWzoyL+lwZKWGrxwaNBIIT08GMGS1swtMIpBsro4jsGJOMR8lxSPIOEufZTIDTIEs4ApfXRfo24ICvEIRPsIv5TuLDLGUGOTw+8Wa5fNzpYaNFQF0DUwiQtXznEtRoqd/0RURqhWGILEtFQVF4OmmiCe++JFGXvq29FSm9p47uRYFPPqw0aWVc+wFhIamowb3TbzobO1GlMJBSfHYmhrCOPIwKSr+Q6FkuywmjAE4qBPStnYRGIKJKZtSojiQf+bHsI5x11bu3KePGVF8wh8RfEIZH0b5tQbxa6Yyy3dFkMgScwY1HKDqTFYNkT2kvk6xqOyIWbPBFpqQznvsM07ZkATi9vqw0bdRc/wlKvU0WwIg3z7Uwfx4Z+8hGNDkWnJfmmu0f5+hyJx7OsbR8gvYWVbXZpGUBP0zxgRdqYwc+6cWciJ0Si+/tBe1Ib8BfU+mUooqAr6UF8Uj0AzBE7ZN3bFXG45NhhByC9hjqlNwbUbF8IvMVywwv3wIaOwLWo2BIkZoxHkS1N1EIxpHkFC0dpIzGsIY159GH6JoWc4gt4Rzcg6FZO5QaQ//uLFo1g2pwZ/e34nPuQybFcIhkYwGcf+vnGsbKtDU3UQI2aNIJ55FgGRH7P7zilzRL5/IsMwdjcIj8AnMdSF/YUZAlUPDfnsd1w1IV+aUOsWkTpq3s1ds3Ghq+HnZupsCttmkkaQLz6JobEqgKHJGE6Nx8A50FYfhk9iWNBYhZ7hKZwYFSml+XsE8xuq8JtPnIu5daGS1w6YaagKgDHN49nXN4aLV81FTFZTeh9NxpWyFopnKvQ/6iEiHl9oW+dIXDF2U43VgaJ4BH6H0FBNKL3hm1uODUWwuKXwNgXCIxChIUXliMSVWe8RAPqQ98m4UUPQ1qB5V+1NVegejqBXby/hZlhOJsxtJ6YLv09CQ1UAB/rHMTARx8q2ehwbnEwJdU7F5bJOHZ2pkEbgIXHDEBTmEQixGAAaq1Jd6VxJegTFDQ1xzlNqCArBmsYqagpmu0cAaIZgcCKOPn3n31av7fzbm5IeQUtN0NWA+3KkuSaIFw5prbFPb6tDQ3UQY9GEkdY8GVOMehaieJAh8BBhABJq4R6BMAQNVQFX/YZe6hrEWV97HGPR1OcmhEaQwSPIRywemoxjMq5knMvrFhEaEp7J2AzpM1QMmvV+PIYh0Hf+7U3VODUew5GByYL0Aa9prg5iWPcANI0gAM6TvaQiCcVVwzkiN8gQeIjYfRfuESiGgNbgMjS6A+cPAAAgAElEQVR0eGASg3q+thmlRFlDbga0u0WEhoRnMp1DabymuSaEock4To5FEfRLRl8mkTm0o3ukoIwhrxFD7OfUhdBSGzIG1ojwUCQmUw1BCSBD4CFxWVt0C9EIOOf6LinpEbgZVykEauu5RR+bbKEhznMzXsU0BFUBHySWDA2J3WJtaPaHhlpqtB3zidEo5tUnU1BFyG0qoRRUQ+A1Yoj9Kr3iurFK+1l4uaJ4kiguZAg8xPAICsgaiisqFJUbN0djleYRZFuohQFIWLwRcS0Bx6whP1QORBO5GS9RQ5CtxbQbRGGb8ATGDY1g9i8QzTVBKCrHgZPjaKtPLvjmGQ7z8+wzVA4Ij0DMhG4wPALNc43EySMoBWQIPESEhAoxBFOWJlwNVQHIKs86Vzh57tQFPXtoyH54fDaODUUwty5UtBzwOlOIamIGTCcrFqK6+ED/hNFkEADm1oUN411InyGvEY31kh5BamhoMq6gmsTiokOGwEPixq48/9CQdXSfiKlm0wmEQG31CMS1BGxaTADJjJ1cM4f6x2MpC1eh1IaTHUine16xl4jqYkXlKQu+qCUAkHfn0XJgbr2WDrt6geYRiEaKI5E4ZEVFXFaNynKieJAh8BBjV16AWGwd1JFsM5E5hTR5bjXtuMTg2CM/3w6kCUV1DDflQ23Ib/SmH6uw0JDAalhFeGgmewSXrWnDTz96Nla1aYZAJACMTCWo4VwJIUPgIUmNIH+PYCqeOsO1ocpdK2qjmE1N1wjsGs4JavM2BNxRgM6H2nAgRSMI+qQZmzufCy01yfYc1qKx9sbqlLkUM5FwwIeLTHN+/T4JdWE/RiKJtL91onjQ/6iHGC0mCvIItMWw2uIRZMscEllD1rCUrKiO7SWA/ENDsqIW9QauDfmMIe3j0UTZj6gsFk01yfBXm2XBv27zInTOqSn7KW25Iqrlxd8ceQTFpzLunjJF7MaVAsRia2jIyLt26xHYZA05tZcA8vcIZJVnNDC5Yp5SNjGDhtIUSsjvQ11Im9ls3flv7GjCxo4mj66sdIhqeaseRhSP2bV1mGHIJRCLG1x2IE04ZA3JqpoxhFNreAS5NZ5LKJlDTrlSGwqY6ggqxxAAyRTLmRwCyoXGaq1aPkKhoZJBhsBD4kURi3V3OaDdHNVBHwI+lnUmgeyQNSQrmT2CfOcWy8UWi8OaWKyqXGtBXQHFZILmmiBaaoIIVshwFlEkKZIDKH20+JBp9ZCkYFuAWJxIDQ0xxtBQFXQhFtt7BAmFZ4wx2w2FcYMWGiqmR+AD51rvmfGoXJSK5ZlCZ2tNRU3oEh6BEIspfbT40P+oh8iGYFu4RmCOmzZU+TE6lTl9VJwzvbJYdSwmA7S00uqgL2ePIKGojo3s8kG0k5iIyhUxi8DM169ZW1AR4kxDDKcRmhBpBMWHDIGHiKyhQjyCiKWyGNCKcLJ6BA4N79yIuvl0IE1kyUbKleRMggTGKmA6mZlKG8zSUBWAyoH+ca3jKhmC4lM5/mUZUozuo2JQh7kArKEqkF0jcBKLFdWxqliQTwdSuch1BKIV9VhUrqisoUpEVBeL6WskFhcfMgQeUoxeQ+ZZBIJGF4Yg4dR0TnHjEeQXGgoWtaBMWwz6x7SRjWQIZi+i39CJkSkwBoQDtGwVG/of9ZB4EUZVmmcRCKpDPkNEdkJ2aEOdrY4AEK2oc0sfdfO6uSAEw95RraiskjSCSkPUxvSORFEd8KXMvCaKAxkCD5EdBNtcsPMI/JKUtTYh4dhiInMdAVAmoaGwMATRlJ+J2YcwBCdGplBdYfrIdFHQnckY+zZjbB9j7A3G2H2MsUbTY7cyxg4yxvYzxi4zHb9cP3aQMfalQs4/0ylGr6FIQkGVJWYa9Gc3BEkjlJ4+mm3nXmNq+OaWhFr8pnMAjDYT5BHMXkT/rPGYjBoSiktCoVu0xwGs5ZyfAeBNALcCAGNsNYDrAKwBcDmAHzLGfIwxH4DbAVwBYDWA6/XnViTJCWWFicXVAatHwLK+ppNQrbjMGspFI1BUDs5R1MpikTnTRx7BrEdUywNI2/QQxaGgO5Nz/hjnXKwILwJo17+/GsC9nPMY5/wwgIMANutfBznnXZzzOIB79edWJMWYUGYXGgr4JMgqzzilzKgjsMkayrZg15qmg7khOf6yeB5B0C8h6JeSoSEKGcxagn7J8ATIIygNxdQI/hrAn/TvFwLoNj3Wox9zOl6ROM0EyAU7sVi0HsikPTh5BFplcRaPIOhHTFZdX3e28Zf5Uhfy4+SY8AgoNDSbESmkxZpwR6SS1RAwxp5gjO2y+bra9JyvAJAB3CMO2bwUz3Dc7rw3MsZeYYy9curUqezvZAZiTCgrskcgYvyZdAJhAKydTxWVZ6wsBsz9htxlDiX0wrlihoYALYVUGBkKDc1uRHiI2kuUhqz/q5zzSzI9zhi7AcA7AbydJ2MRPQAWmZ7WDuCE/r3Tcet57wRwJwBs2rRpVtbTy0VIH9WGead+jKJXUCadQHaYR5BwkTUkFt2JuGwMF8+ECD8FitwfRwjGPr3tBTF7EZlD9DmXhkKzhi4H8EUA7+KcR0wPPQDgOsZYiDHWCWAFgG0AXgawgjHWyRgLQhOUHyjkGmYyyVz+AsTiRHpoSIRg4hk9AocWEwrP2hMo1+E04hzF7DUEJA1BbchPueWznCY9NESdR0tDoX7WDwCEADyu34gvcs5v4pzvZoz9BsAeaCGjmznnCgAwxm4B8CgAH4C7Oee7C7yGGYsxoSzP9NG4rCKh8LSsIeERZAoNOYnFWmgo8/4g17nFwhAUs44ASBoCCgvNfoTnSaGh0lDQ/yrnfHmGx74B4Bs2xx8G8HAh550tFDqhbMoynUzgdxUachKLs+f7G1PKXGYOGaGhIovFos0ECcWzH9FmgsTi0kCVxR4im/r9ZEr1dCKSEG15rRqBm9CQ04QyF3UEwfxCQ0UXi4VHQKmjs55G8ghKChkCD4mbduP5eAVOM1wNsThDyMm56Zy7OgLAfWioFHUEgNkjoMVhttNYRemjpYQMgYeYs4XyKSpzCg0ZGoGcPWson6ZzuY6rLFUdQW2QDEGlYGgEJBaXBDIEHmJe/PMZYO/kEYiddyYR2qkFtpvmcGInPhl3WUeglK6OACCNoBIwNIIAGf1SQIbAQ0TWEJBfCqkxuN5aWWx4BBlCQ8bweqtHoGb1CEJ+HyQGRLO0ujbOVarQEGUNVQxntDfiA1s6sKWz2etLmZXQHeQh5hh+QaGhgENBmcNriiZwQKoBUlUOlbtbsAM+KaMYbUaco5iDaYCkASCPYPZTFfThX65d5/VlzFrII/AQ2dTXJ59W1NlCQ04LtdkLMLe3SKZ5Zv+zCPqkjBqEGfHeil1HIOoZaskjIIiCIEPgIXFFRdivLeJ5hYb00Iy12jKYpY7A7CmYxWKRuZSt1xCgtYuIK25DQyJ9tDShoXoyBARREGQIPERWOML6bj4fsXgqbl9HYIjFDq+Zkq2kmAVr9wt2wMfcewSixUSRPYKVbXW4Ym0bNlPcmCAKgrZSHiKrqhHWyUcjiBgaQW4tJsy1A+bMImEgXIWGXExBM15XLY1YXB30444PnVXU1ySISoQ8Ao/gnCOhcGMRz88jUBDyS2mhnICUeR5Bikhteo6cS2goB7E4YTSdoz83gihH6M70CLHohnVDkG9lsV1b3oA/W2hIOxdj9kVtbgq/gr4cPIISpY8SBFEcyBB4hFiMqw2NIF9DkB7dE4VbTnMOxAJeFfClZA3JORR+BXxSSh1EJkpVR0AQRHEgQ+ARIjYvPIJ8htNMJWTb3isiayieJWuoKuBLOa8hFruqI2CujVeiRHUEBEEUB7ozPUJU/QqNIF+x2C40JBbybB5BOOBL0QhEeMqNRxD051BQVqI6AoIgigPdmR5h7MoLSh9VDI/CTLasIbH4VwV9KVlDuYRwAjloBKWqIyAIojiQIfAIc5weyE8sjisqQjZzgANGHUHmrKEqi0eQq1jsViMoVR0BQRDFge5Mj0goVo8gD0Mgq7Zxd8YY/BLLWkdQFfBBVpNDcRTdQGQbVQnk5hHIqgrG3KWlEgQx/ZAh8AhZsWoEuYeG4rKKoI1HAGgLtZPuIHboYUsxWzLf312LiVzEYqohIIjyhe5OjxCLaDJrKL/QkFO4xe9jjqGbhBEaSu1JlMuQ+dxCQyqljhJEGUOGwCOSGkFmYTfja2TwCDIVfMmm0BCQNAyyERpyoRH4nUNPadepZJ9xQBCEd5Ah8Aix6IqCsHzSR+OKsyHw+5hz91FhhIKp3khS1C1yiwmVO14nQRDeQ3enRyQc4vS5EHMQi4HMYm5CtYalUj0Ct5XFmSagmZEVtehjKgmCKB50d3qENX00n8riuGyfPgroC7WjWJx6bvE8YYzc1xG4b0NNGgFBlC9kCDzCGqfPVSzmnGcMDWnzAtxpBIZHkEPhl6gsFqmnmUionGoICKKMobvTIxKWOH0ix/RRMXfYMWtIkhxTUo2sIUsNg7gmd6MqxYjN7IZAJrGYIMoaMgQekUwfzTxW0gkh1Dp6BH7JuemcNXVVNww5jaoUje1c6AQJhVOfIYIoY+ju9Ai5wO6jYgF2EouDPuaqDbV2bt0jyFEjML9WJmRVdZWJRBCEN5Ah8Ahza2a/xHLOGjIMgVP6qJShjsCh4Z0xqtJN1pBftLp24xFQaIggyhkyBB5h7vTp9+VuCGJZDEGmFhDp7S1EryE9NORi9x4yPAIXYrFCYjFBlDN0d3qEeVB8IMPu3QlDI3CqI8jQdE4s/CL1VDwvl9nCxjhMFxqBnKEVBkEQ3kN3p0eYF91MVcDOv5/FI8jSYkJiyfCO8ARymS1siMWuNAKqIyCIcoYMgUekhoacUz2dyCYWZzIuCVWFX9cmAFOLCdV9HUHOWUNUWUwQZQvdnR6RHAIj6bMDiisWBzP0ApIVjoDE0jJ/ZFWFT2JgzN1gGvPvZkILDZFHQBDlChkCj0gWb2lica4TyrJmDWVpOuf3ScnZxkZoiLvO7gn63YvFWmiI/tQIolyhu9MjEkpy920nFv/+1R50nZpw/P1YlirgbE3nAj5mhGvMYrFbQ5BLHUFCUV0NuyEIwhvIEHiErHAjXGK3e//i797AvS93O/6+yNbJ2HTOMTSkdQMV5xfnVnTtwA3id91oBNR0jiDKm4IMAWPsa4yxNxhjOxhjjzHGFujHGWPsNsbYQf3xM02/cwNj7ID+dUOhb2CmYh7faO0LJCsqEgpHNKE4/n7WFhM+Z91BLMxi0RfnFp6CG3LJGkpQ+ihBlDWF3p3f5pyfwTnfAOBBAP+gH78CwAr960YAdwAAY6wZwD8C2AJgM4B/ZIw1FXgNM5KEaXyjddEWi2ss4bzIZssaCmTIRBLdQEW4Rpw7l7kBQX+OoSEyBARRthR0d3LOx0w/1gAQq9nVAH7ONV4E0MgYmw/gMgCPc86HOOfDAB4HcHkh1zBT0frvaP/9PomlLNpR3QBE5QweQVaxWKsstmsTLbqBGh6BkTXEXTWcA3LMGlLdaw8EQUw//kJfgDH2DQAfATAK4CL98EIA5gB3j37M6bjd694IzZtAR0dHoZdZdpjbLvh9UopGENMNQEaPIItYbG4TbQ33iG6gdllDrkNDwiOQ3bShpqwhgihnst6djLEnGGO7bL6uBgDO+Vc454sA3APgFvFrNi/FMxxPP8j5nZzzTZzzTXPmzHH3bmYQ1tCQudeQMACxAj0CcR4rohtoQEpNAZXzEItjbkJD1H2UIMqarB4B5/wSl6/1SwAPQdMAegAsMj3WDuCEfvxCy/GnXb7+rMKcs++XJMiKbDwmQkJRFx5BpqwhQN+xB+3PbXgEpgllrusIjNfPbAjEAB2qLCaI8qXQrKEVph/fBWCf/v0DAD6iZw+dA2CUc94L4FEAlzLGmnSR+FL9WMVhFlCtYnFOHoGjWKwLwTaCcUJJbTFhnlnsNs3TbR2BuZUGQRDlSaEawTcZYysBqACOArhJP/4wgCsBHAQQAfAxAOCcDzHGvgbgZf15X+WcDxV4DTMSsyGwpo+KFtOxDLvtuKwJvpLDDj7TQi2rHOGABMaYNgvBKCgrftZQspUGGQKCKFcKMgSc8/c4HOcAbnZ47G4Adxdy3tmAefdtLSgT9QPZDEGmlExrQ7mUcysq/CF/8tymeQRuQ0PieU7jMAUidEShIYIoX+ju9IhUj8AiFuuLZ6aCsoSiOgrFQHLHblfwlTBlB5nbW+RSAcwY0xrbZdEIRGgqkOFaCYLwFro7PSKR0mJCSpkvbKSPZvIIshiCgFEjYOMRqMkQkNkbSai5FX5p2kaW0JAxd4FCQwRRrpAh8AhzFW/AxwzBFjCJxRk8gpisOgrFQDJ0Y6sRKOawVFKfUHIoKAPEOEx3hoDqCAiifKG70yNSCsqkVI/ASB/NohE4pY4CpoIvu9CQaecfMM1CyHWATDBDYzvzuQASiwminCFD4BEJ07AWq1gsPIK4rNq2iBCPZQwNWYrFzKTUMJjCUrkOkAn4JMSzVBYbHgGJxQRRttDd6RHmYS0Bn5SS72/WBpx0gmyN3AKWYrHU302e228KS+UaGgr6naegma9TnIcgiPKEDIFHmD0Cn5Q6ocycLeTUbyibWCwWeruFWja1fAiYwlJ5icVZsoaojoAgyh8yBB6hTe1KjdOLMFCqR2AvGMeziMXJ7qBOoaH0rKFcWkwAmYffJM9FdQQEUe7Q3ekR1swdAIZXYF78nUJD2TQCax8hM6n6hJRXiwnAXWgoW5dUgiC8h+5Oj0gpKLO0gzY3m3PyCGLZxOKMoaHkgh8wtZjIZTCNOIfrOgIKDRFE2UKGwCOs1b3asfRmc04dSBNK5tCQdR6xgHOut5JwCA3l4hG4qCwWNQpUR0AQ5QvdnR5h7v0vFt9kaMiFRuCysti6Y09YdujmjKVcJ4llmotsPR9NKCOI8oUMgQdwzi0FZamzg2NusoayiMWGIVBTF2rrDl3rPpr7YBpxDvehIfpTI4hyhe5ODzBSKqVUsVgs0jFZhdhAO80tzlpQJuYRyPYegbmgLKGoSeNU5DqCpOEhj4AgyhUyBB5g7b9jbRkdS6hoqAoY39uR3RCkGpfkuVOzeMSYTBGWysUjcNViwmg6R39qBFGu0N3pAdb+O9Z4flRWkobAKX00S2Wx2IFbY/iyseCbx2SqxvGcms65EYupspggyh4yBB4gwjVO6aOxhIp63RDYzSQQYRx3vYasoSE15XG/LvjmUwEc8LsRi6mOgCDKHbo7PcBuVw6YQkNZPIJsg+sBQJIYfFL6vIBkWMrUYkJV86oADvikrC0mrFlKBEGUH2QIPCBtVy40ApNYXG8YgnSPINvgeoFdemda1pBeR2A1Tm7ITSymPzWCKFfo7vQAY5fsT7ahNh+PJhTUh0VoyMYjEIYgy/hH8xhKgTVjSaSA5tMuOhexmOoICKJ8IUPgAdYwTHKsZNIjqAn64JOYvUfgMu5uN0HMLmNJVnle7aIDPgkqt+9nZD0faQQEUb7Q3ekB1rh5MjSUrCwOB3wI+yXb9NGEPgwmm0dgLhZLnjt1wddGVZrSR3PMGjK/HztkVQVjuWUjEQQxvZAh8ABZtWYNJTN8EooKReUI+SWEAj4HsVjzErKGhnzpMfxkaMhUR6CoecXyhSHLpBNoRWr0Z0YQ5QzdoR6Q3JUnF2NA6zUkFv5QQELIL9mmj8ZcisVBv+ToEYgduk9iUHnyNXOpLA5lmIsskBWVaggIoswhQ+AByWrb1PTRhMKNPkMhvw8hv2TvEQhj4SI05KQRWIvZhCida0GZdt2ZPILcpp4RBDH90B3qAcZi7LcWlKmI6ot8OCAhHPBlTB/NKhb7pOzpo/rCLwxQbqMqpZTrsSOhcqohIIgyhwyBBxihIckiFtt4BHbpo2Jxz64RpHsEdk3nAGBKP29OWUNuQ0OkERBEWUN3qAdY2y6YQywxU9gn5HfwCHIQi9ObzqWmc4rdujAEuYSGgkIsljNkDeU47IYgiOmHDIEHJPv6pPcaEuJwOOBDKJBZI8gmFvt9zEg1TZ7bkj4qpWoEuYSGgi48Ai00RH9mBFHO0B3qAWm5/KLXkDlrSPcI7EJDMbeVxTbpo9a20H6LR5BfHUG20BB5BARRzpAh8ADrYpycL6ympo8GpIxicbasIfvQUKoREueOxoUhKLJYrHDqM0QQZQ7doR5gDIfxJ3P5tePJ0FDI70PY77OvLHbZtiFgExpKOHQ+jeYjFgtDkMkjUFXKGiKIMocMgQcks4YsYrGa9AjCGT0Cd2Kx3zSYXmAYIcleLM5l0Q4aoaHMYjFpBARR3tAd6gGOvYbsCsrsuo8q7jQCu+6g1nkEwhgls4aKKxbHSSMgiLKHDIEHWHsNJUNDyYKyZPpo/llDAZ9N07m0Pke6RpBIrW1wg9FrKINGIFNlMUGUPXSHekDCsitnjBlD5A2PIOBDOKBl/ahq6mKerCzOvGj7M3kEpnkEQFIjyKuyOKNGQHUEBFHuFMUQMMY+zxjjjLFW/WfGGLuNMXaQMfYGY+xM03NvYIwd0L9uKMb5ZxrWCWWAPkTeJn0USB9XGVc4gj4JjGVeYIN2LSYsTeeEQZiK51FQ5qaOQOFUWUwQZY6/0BdgjC0C8A4Ax0yHrwCwQv/aAuAOAFsYY80A/hHAJgAcwHbG2AOc8+FCr2MmISscPolBMi26okFcqiHQFtCYrKAq6DOeG5fVrPqA+TXNiN4/woiI1M6onLtYbIjcWUND5BEQRDlTjK3adwH8b2gLu+BqAD/nGi8CaGSMzQdwGYDHOedD+uL/OIDLi3ANM4qEjYAqZgfHEgpCfm23Hw44eQSKK0MQsGlDbe39Y2QNiTqCvCqLMw2moToCgih3CrpDGWPvAnCcc/665aGFALpNP/fox5yOVxQJm5RKv178FZNVwxMQ/1pnEsRlNatQDGhtruOKCs6TC3XC0vsnrY4gH7E4WxtqyhoiiLIma2iIMfYEgDabh74C4MsALrX7NZtjPMNxu/PeCOBGAOjo6Mh2mTMKWU0f1hKQmL6zVhDSPYFQQISGLB6By9CQMQvZ1ApaK/BK9wjyyhqSslcWU9M5gih/shoCzvkldscZY+sAdAJ4XY83twN4lTG2GdpOf5Hp6e0ATujHL7Qcf9rhvHcCuBMANm3a5Bx7mIHYDWvx+yQo+uzgsG4AwkIsttQSaB5F9sVVhGS0oi4Y35sXe2sb6lzEYklitjpE6rVS+ihBlDt536Gc852c87mc8yWc8yXQFvkzOed9AB4A8BE9e+gcAKOc814AjwK4lDHWxBhrguZNPFr425hZaHN80zUCTSxWjGwh4RFELdXFMVlF0O9DNuxCN9awlJE1lFBSRGS3BGxSVM2QISCI8qfgrCEHHgZwJYCDACIAPgYAnPMhxtjXALysP++rnPOhEl1D2aLN8U1dHAOSJuwmFLNGYO8RxJUcQ0OmhdoaljLqCOJKTt6AIOhPT1E1I6ucKosJoswpmiHQvQLxPQdws8Pz7gZwd7HOOxOxC+34JKaPqlSMbKGwoRFYxWIFITdisU0voPTQkK4RyIoRisqFgM9+ZkLK+cgjIIiyhu5QD7ALl2hjJTliCRuPIG+xmBnnczq3EHyt2URuCdqMwzSToO6jBFH2kCHwALu2Czmlj7os0rIbHGM9t/n7XBrOGefwO2sEisrBeW4zDgiCmH7oDvUA26whiRnzCKxisdUjSMg85/RR87nNC3OqXpD7zj2TWGydxEYQRHlChsADtCIra2go2WsoPX003SNwkzUkFmBznr9s0ScCDkbBLUGf5Di8PjmbmQwBQZQzZAg8wK7ISmsx4ZQ+aqMRuBBggzYegaymegSSxCC043xCOAF/+lxk41yiuR6JxQRR1tAd6gFOoaGEwhFNqIYBcEofjbltOmcrFtvrE+IaciXoY45N5+JGaIj+zAiinKE71APs0ke1NtSaRyDSR32SNqfALn00mItYLKfWEaRlLImW1Hks2Jk0AtHwjnoNEUR5Q4bAA6zhGcDUfdSUNQRoXkHUpsVETumjqnMdgXbu/D0CN4aAPAKCKG/oDvWAhMIR8KeLxVMJBZzDYgjSB9jnWlls9gicahiAPMViv4S4Q2VxciwmeQQEUc6QIfAAu9bMfolhIiYDgBEaEt+b00dFY7qgz0XWkCTE4tTfT9MI9OdZM5ncoGUNKbaPJcdi0p8ZQZQzdId6gH3WkIRJ3RBYPQJzQZkxuN6FRxD0i6ZzlnkENmEpILfOowJREW0H1REQxMyADIEHOGUNiVB+yFQjEPSn9vLJxRA4NZ2zhmrE8/JZsDNqBFRHQBAzAjIEHmA/jyC5WIr0Ue371NCQSMl0kzXkt2sxYeeN6J5APvn+mVpMUB0BQcwM6A71ALvWzObF0uwRhP1SSmWxYQhyajrn3GICSBqMvNpQ+yTHCWVGHQFpBARR1tAd6gEJm3kEZsNg9Qii+YaGJBuPQE2vYRA/5xPCCWasLKbQEEHMBMgQTDOcc60OwKG6F7BJH7UTi11kDYkUVdk6j8DBCOXVYiKDWCyylaiOgCDKG7pDpxlFtS+yMqeTWtNH43l6BGKBTxlVqarpBWVSYQVlIqXVSsJIHyWPgCDKGTIE04yxOFonlJnF4kzpo4r7Ii3rPAKn+QD+AgrK7GYeCJKhIfozI4hyhu7QaUZU21q7h5qLucxicaiA9FGf3llULMhOef1Gi4k8FmxhtGwNgUp1BAQxEyBDMM3IDuES82IZNonF1spi4RGEXBgCIDXP3ymvX4Sl8mkOJ3b7dplDCaPpHP2ZEUQ5Q3foNJNwaM3sd0gfdawsdiEWA8IQaAuy7JDOmawszq/7KABbwdioI/CTR0AQ5QwZgmkmoTiFhhzSR/0+yCo3FtVcQkOAyOpR9XPbewTCCOU3qjJ95oFAdD2lOgKCKG/oDp1mZAex2OwRhHqYnSIAAAqfSURBVM0FZbpRiBuLeW4dPbURmCI0ZO+NJOcR5FdHYL4+M6LrKdUREER5Q4ZgmnEKDYnFkrHUhVNoAWImQe4eQXKmsLM+ISqL8+s+CmQTi+nPjCDKGbpDp5mEw9Qu0d4h5JfAmDlMpI+r1Fs9x3JoMQFoRkUsyAmH3j9GZXGJxGKqIyCI8oYMwTRzbGgSANDWEE45LuLoZqFY+1k7HrN4BCGXYrHfJmvIaR5BXqMqM6WPUh0BQcwI6A6dZt7oGYVfYjh9fn3KcbErN6eOaj8LjyD/0FDCWkfgVFCW5zwC7bpssoZUFYzl18yOIIjpgwzBNPNGzyhWttWltJEAkrtxJ49ApJDmLhYns4acmsAVMo8gk0aQUDjVEBDEDIDu0mmEc443ekZwRntj2mMBk0ZgRhgGs0cgMfdhnNSCMocaBiNrKA+xOGNoKH0IDkEQ5QcZgmnk6GAEY1EZZ7Q3pD0mFmGrpyBCRUIsdju43nhdiZlCQ/ZCtdFioshisaymdzolCKL8oLt0Gnm9ZwQAbA2BL4tHYE4ftRajZSJomiCWrGFwqCMoxBDYeARx8ggIYkZAhmAa2dkzipBfwmnz6tIeEwtmyCIWhyweQUxWEfS7yxgCtMXdaDrn0AQuWVlcSB2BfYsJqiomiPKnYu5Szjluf+og9vaOeXYNb/SMYvWCetsFVyyYYcsiL36OpXgE7nfZKRqBQxO4gNFrKA+PwO/cYsJuPjJBEOVHxRiCFw4N4tuP7sfXHtzjyfkVlWPXiVGstxGKgeweQVROZg3lohEEUkJDDh6BMby+yFlDKqcaAoKYAVTMXfrjrV0AgOcPDWLX8dFpP/+hUxOIxBWsW5iuDwDZ00dTPIJcDIFZLHZoQ50Ui/MvKLMVi5X0aWgEQZQfFWEI3jw5jqf2n8Lfnt+JmqAPd+lGYTp5vVsTitcvcjAEbtNHc/UIfJLhCTi1oTZCQwV4BLZN52zmIxMEUX4UdJcyxv6JMXacMbZD/7rS9NitjLGDjLH9jLHLTMcv148dZIx9qZDzu+WurV0IByR88sLluG5zB/74Ri9OjExNx6kNdh4fRU3Qh6WttbaP+43KYgePQKSP5pg15PdJiCuWpnMOLSbyKf4y5hE4VBZT1hBBlD/F2K59l3O+Qf96GAAYY6sBXAdgDYDLAfyQMeZjjPkA3A7gCgCrAVyvP7dk9I9H8YfXTuB9Zy1Cc00QH3vrEgDAT/9yuJSnTeP1nlGsXdgAySFUkuw1lPqRSBJD0Celpo/m4BEEzU3nVPumc4XMLBbjMJ3EYtIICKL8KdVdejWAeznnMc75YQAHAWzWvw5yzrs453EA9+rPLRk/f/4oEqqKj5/XCQBob6rGX62bj19t68ZYNFHKUxvEZRV7T4xh/SJ7oRgwicU2i7w2tzhZUJbL4ur3ScZcAKc21IECCsqA1FoFM3HSCAhiRuAvwmvcwhj7CIBXAPwvzvkwgIUAXjQ9p0c/BgDdluNbinANtkTiMn7x4lFcunoelrTWGMf/9vyleOD1E7jie1tRHXSfk58vssoRV1RHoRhILsahQPr1hAI+/G57D547MICjQxG8bUWr63MHfBIm4wre8Z1nMBzRDF/6LIT8u4+K3//1K914cl9/yvHu4Qg2LW7O6zUJgpg+shoCxtgTANpsHvoKgDsAfA0A1//9DwB/DcBuG8hh74GkB5e1894I4EYA6OjoyHaZtoxHZZy3vBV/fd6SlOPr2hvwuUtOw/6T01dTcNbiJlywco7j4zUhP75w2UpcuW5+2mOfvHAZth8dAgCsmFeL95zZ7vq87zxjPrqHI+Bc+29ub6pGfTj1Y9+ytBmfeNtSrLZ0RHXLLRctN6qmzayYV4urzliQ12sSBDF9MLFAFPxCjC0B8CDnfC1j7FYA4Jz/q/7YowD+SX/qP3HOL9OPpzzPiU2bNvFXXnmlKNdJEARRKTDGtnPON2V7XqFZQ+bt67UAdunfPwDgOsZYiDHWCWAFgG0AXgawgjHWyRgLQhOUHyjkGgiCIIjCKFQj+DfG2AZo4Z0jAD4BAJzz3Yyx3wDYA0AGcDPnXAEAxtgtAB4F4ANwN+d8d4HXQBAEQRRA0UJDpYRCQwRBELkzLaEhgiAIYuZDhoAgCKLCIUNAEARR4ZAhIAiCqHDIEBAEQVQ4MyJriDF2CsDRAl6iFcBAkS5nplCJ7xmozPddie8ZqMz3net7Xsw5d25poDMjDEGhMMZecZNCNZuoxPcMVOb7rsT3DFTm+y7Ve6bQEEEQRIVDhoAgCKLCqRRDcKfXF+ABlfiegcp835X4noHKfN8lec8VoREQBEEQzlSKR0AQBEE4MKsNAWPscsbYfsbYQcbYl7y+nlLBGFvEGHuKMbaXMbabMfZZ/XgzY+xxxtgB/d8mr6+12OizsF9jjD2o/9zJGHtJf8+/1tudzyoYY42Msd8yxvbpn/m5s/2zZox9Tv/b3sUY+xVjLDwbP2vG2N2MsX7G2C7TMdvPlmncpq9vbzDGzsz3vLPWEDDGfABuB3AFgNUArmeMrfb2qkqGDG1M6OkAzgFws/5evwTgz5zzFQD+rP882/gsgL2mn78F4Lv6ex4G8HFPrqq0fB/AI5zzVQDWQ3v/s/azZowtBPAZAJs452uhtbC/DrPzs/4vAJdbjjl9tldAm/WyAto0xzvyPemsNQQANgM4yDnv4pzHAdwL4GqPr6kkcM57Oeev6t+PQ1sYFkJ7vz/Tn/YzANd4c4WlgTHWDuCvANyl/8wAXAzgt/pTZuN7rgfwNgA/AQDOeZxzPoJZ/llDm51SxRjzA6gG0ItZ+Flzzp8FMGQ57PTZXg3g51zjRQCNlmFhrpnNhmAhgG7Tzz36sVmNPjJ0I4CXAMzjnPcCmrEAMNe7KysJ3wPwvwGo+s8tAEY457L+82z8zJcCOAXgp3pI7C7GWA1m8WfNOT8O4N8BHINmAEYBbMfs/6wFTp9t0da42WwImM2xWZ0ixRirBfA7AP+Tcz7m9fWUEsbYOwH0c863mw/bPHW2feZ+AGcCuINzvhHAJGZRGMgOPSZ+NYBOAAsA1EALi1iZbZ91Nor29z6bDUEPgEWmn9sBnPDoWkoOYywAzQjcwzn/vX74pHAV9X/7vbq+EvBWAO9ijB2BFva7GJqH0KiHD4DZ+Zn3AOjhnL+k//xbaIZhNn/WlwA4zDk/xTlPAPg9gLdg9n/WAqfPtmhr3Gw2BC8DWKFnFgShiUsPeHxNJUGPjf8EwF7O+XdMDz0A4Ab9+xsA3D/d11YqOOe3cs7bOedLoH22T3LOPwjgKQDv1Z82q94zAHDO+wB0M8ZW6ofeDm02+Kz9rKGFhM5hjFXrf+viPc/qz9qE02f7AICP6NlD5wAYFSGknOGcz9ovAFcCeBPAIQBf8fp6Svg+z4PmEr4BYIf+dSW0mPmfARzQ/232+lpL9P4vBPCg/v1SANsAHATw3wBCXl9fCd7vBgCv6J/3HwA0zfbPGsA/A9gHYBeAXwAIzcbPGsCvoOkgCWg7/o87fbbQQkO36+vbTmhZVXmdlyqLCYIgKpzZHBoiCIIgXECGgCAIosIhQ0AQBFHhkCEgCIKocMgQEARBVDhkCAiCICocMgQEQRAVDhkCgiCICuf/A7A33Y00JYGxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "actor_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.01}\n",
    "critic_kwargs = {'hidden_sizes' : [100,], 'learning_rate' : 0.01}\n",
    "agent = SACAgent(env, actor_kwargs=actor_kwargs,\n",
    "        critic_kwargs=critic_kwargs, batches=50)\n",
    "\n",
    "\n",
    "# 训练\n",
    "episodes = 100\n",
    "episode_rewards = []\n",
    "for episode in range(episodes):\n",
    "    episode_reward = play_qlearning(env, agent, train=True)\n",
    "    episode_rewards.append(episode_reward)    \n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "# 测试\n",
    "episode_rewards = [play_qlearning(env, agent) for _ in range(100)]\n",
    "print('平均回合奖励 = {} / {} = {}'.format(sum(episode_rewards),\n",
    "        len(episode_rewards), np.mean(episode_rewards)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ywujXzawzmUo"
   },
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Acrobot_v1.AC.201901302315.fail.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
